drivers/staging/media/ipu3/ipu3-css.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (C) 2018 Intel Corporation
   3
   4 #include <linux/device.h>
   5 #include <linux/iopoll.h>
   6 #include <linux/slab.h>
   7
   8 #include "ipu3-css.h"
   9 #include "ipu3-css-fw.h"
  10 #include "ipu3-css-params.h"
  11 #include "ipu3-dmamap.h"
  12 #include "ipu3-tables.h"
  13
  14 /* IRQ configuration */
  15 #define IMGU_IRQCTRL_IRQ_MASK   (IMGU_IRQCTRL_IRQ_SP1 | \
  16                                  IMGU_IRQCTRL_IRQ_SP2 | \
  17                                  IMGU_IRQCTRL_IRQ_SW_PIN(0) | \
  18                                  IMGU_IRQCTRL_IRQ_SW_PIN(1))
  19
  20 #define IPU3_CSS_FORMAT_BPP_DEN 50      /* Denominator */
  21
  22 /* Some sane limits for resolutions */
  23 #define IPU3_CSS_MIN_RES        32
  24 #define IPU3_CSS_MAX_H          3136
  25 #define IPU3_CSS_MAX_W          4224
  26
  27 /* minimal envelope size(GDC in - out) should be 4 */
  28 #define MIN_ENVELOPE            4
  29
  30 /*
  31  * pre-allocated buffer size for CSS ABI, auxiliary frames
  32  * after BDS and before GDC. Those values should be tuned
  33  * to big enough to avoid buffer re-allocation when
  34  * streaming to lower streaming latency.
  35  */
  36 #define CSS_ABI_SIZE    136
  37 #define CSS_BDS_SIZE    (4480 * 3200 * 3)
  38 #define CSS_GDC_SIZE    (4224 * 3200 * 12 / 8)
  39
  40 #define IPU3_CSS_QUEUE_TO_FLAGS(q)      (1 << (q))
  41 #define IPU3_CSS_FORMAT_FL_IN           \
  42                         IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_IN)
  43 #define IPU3_CSS_FORMAT_FL_OUT          \
  44                         IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_OUT)
  45 #define IPU3_CSS_FORMAT_FL_VF           \
  46                         IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_VF)
  47
  48 /* Formats supported by IPU3 Camera Sub System */
  49 static const struct imgu_css_format imgu_css_formats[] = {
  50         {
  51                 .pixelformat = V4L2_PIX_FMT_NV12,
  52                 .colorspace = V4L2_COLORSPACE_SRGB,
  53                 .frame_format = IMGU_ABI_FRAME_FORMAT_NV12,
  54                 .osys_format = IMGU_ABI_OSYS_FORMAT_NV12,
  55                 .osys_tiling = IMGU_ABI_OSYS_TILING_NONE,
  56                 .bytesperpixel_num = 1 * IPU3_CSS_FORMAT_BPP_DEN,
  57                 .chroma_decim = 4,
  58                 .width_align = IPU3_UAPI_ISP_VEC_ELEMS,
  59                 .flags = IPU3_CSS_FORMAT_FL_OUT | IPU3_CSS_FORMAT_FL_VF,
  60         }, {
  61                 /* Each 32 bytes contains 25 10-bit pixels */
  62                 .pixelformat = V4L2_PIX_FMT_IPU3_SBGGR10,
  63                 .colorspace = V4L2_COLORSPACE_RAW,
  64                 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
  65                 .bayer_order = IMGU_ABI_BAYER_ORDER_BGGR,
  66                 .bit_depth = 10,
  67                 .bytesperpixel_num = 64,
  68                 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
  69                 .flags = IPU3_CSS_FORMAT_FL_IN,
  70         }, {
  71                 .pixelformat = V4L2_PIX_FMT_IPU3_SGBRG10,
  72                 .colorspace = V4L2_COLORSPACE_RAW,
  73                 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
  74                 .bayer_order = IMGU_ABI_BAYER_ORDER_GBRG,
  75                 .bit_depth = 10,
  76                 .bytesperpixel_num = 64,
  77                 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
  78                 .flags = IPU3_CSS_FORMAT_FL_IN,
  79         }, {
  80                 .pixelformat = V4L2_PIX_FMT_IPU3_SGRBG10,
  81                 .colorspace = V4L2_COLORSPACE_RAW,
  82                 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
  83                 .bayer_order = IMGU_ABI_BAYER_ORDER_GRBG,
  84                 .bit_depth = 10,
  85                 .bytesperpixel_num = 64,
  86                 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
  87                 .flags = IPU3_CSS_FORMAT_FL_IN,
  88         }, {
  89                 .pixelformat = V4L2_PIX_FMT_IPU3_SRGGB10,
  90                 .colorspace = V4L2_COLORSPACE_RAW,
  91                 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
  92                 .bayer_order = IMGU_ABI_BAYER_ORDER_RGGB,
  93                 .bit_depth = 10,
  94                 .bytesperpixel_num = 64,
  95                 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
  96                 .flags = IPU3_CSS_FORMAT_FL_IN,
  97         },
  98 };
  99
 100 static const struct {
 101         enum imgu_abi_queue_id qid;
 102         size_t ptr_ofs;
 103 } imgu_css_queues[IPU3_CSS_QUEUES] = {
 104         [IPU3_CSS_QUEUE_IN] = {
 105                 IMGU_ABI_QUEUE_C_ID,
 106                 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
 107         },
 108         [IPU3_CSS_QUEUE_OUT] = {
 109                 IMGU_ABI_QUEUE_D_ID,
 110                 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
 111         },
 112         [IPU3_CSS_QUEUE_VF] = {
 113                 IMGU_ABI_QUEUE_E_ID,
 114                 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
 115         },
 116         [IPU3_CSS_QUEUE_STAT_3A] = {
 117                 IMGU_ABI_QUEUE_F_ID,
 118                 offsetof(struct imgu_abi_buffer, payload.s3a.data_ptr)
 119         },
 120 };
 121
 122 /* Initialize queue based on given format, adjust format as needed */
 123 static int imgu_css_queue_init(struct imgu_css_queue *queue,
 124                                struct v4l2_pix_format_mplane *fmt, u32 flags)
 125 {
 126         struct v4l2_pix_format_mplane *const f = &queue->fmt.mpix;
 127         unsigned int i;
 128         u32 sizeimage;
 129
 130         INIT_LIST_HEAD(&queue->bufs);
 131
 132         queue->css_fmt = NULL;  /* Disable */
 133         if (!fmt)
 134                 return 0;
 135
 136         for (i = 0; i < ARRAY_SIZE(imgu_css_formats); i++) {
 137                 if (!(imgu_css_formats[i].flags & flags))
 138                         continue;
 139                 queue->css_fmt = &imgu_css_formats[i];
 140                 if (imgu_css_formats[i].pixelformat == fmt->pixelformat)
 141                         break;
 142         }
 143         if (!queue->css_fmt)
 144                 return -EINVAL; /* Could not find any suitable format */
 145
 146         queue->fmt.mpix = *fmt;
 147
 148         f->width = ALIGN(clamp_t(u32, f->width,
 149                                  IPU3_CSS_MIN_RES, IPU3_CSS_MAX_W), 2);
 150         f->height = ALIGN(clamp_t(u32, f->height,
 151                                   IPU3_CSS_MIN_RES, IPU3_CSS_MAX_H), 2);
 152         queue->width_pad = ALIGN(f->width, queue->css_fmt->width_align);
 153         if (queue->css_fmt->frame_format != IMGU_ABI_FRAME_FORMAT_RAW_PACKED)
 154                 f->plane_fmt[0].bytesperline = DIV_ROUND_UP(queue->width_pad *
 155                                         queue->css_fmt->bytesperpixel_num,
 156                                         IPU3_CSS_FORMAT_BPP_DEN);
 157         else
 158                 /* For packed raw, alignment for bpl is by 50 to the width */
 159                 f->plane_fmt[0].bytesperline =
 160                                 DIV_ROUND_UP(f->width,
 161                                              IPU3_CSS_FORMAT_BPP_DEN) *
 162                                              queue->css_fmt->bytesperpixel_num;
 163
 164         sizeimage = f->height * f->plane_fmt[0].bytesperline;
 165         if (queue->css_fmt->chroma_decim)
 166                 sizeimage += 2 * sizeimage / queue->css_fmt->chroma_decim;
 167
 168         f->plane_fmt[0].sizeimage = sizeimage;
 169         f->field = V4L2_FIELD_NONE;
 170         f->num_planes = 1;
 171         f->colorspace = queue->css_fmt->colorspace;
 172         f->flags = 0;
 173         f->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
 174         f->quantization = V4L2_QUANTIZATION_DEFAULT;
 175         f->xfer_func = V4L2_XFER_FUNC_DEFAULT;
 176         memset(f->reserved, 0, sizeof(f->reserved));
 177
 178         return 0;
 179 }
 180
 181 static bool imgu_css_queue_enabled(struct imgu_css_queue *q)
 182 {
 183         return q->css_fmt;
 184 }
 185
 186 /******************* css hw *******************/
 187
 188 /* In the style of writesl() defined in include/asm-generic/io.h */
 189 static inline void writes(const void *mem, ssize_t count, void __iomem *addr)
 190 {
 191         if (count >= 4) {
 192                 const u32 *buf = mem;
 193
 194                 count /= 4;
 195                 do {
 196                         writel(*buf++, addr);
 197                         addr += 4;
 198                 } while (--count);
 199         }
 200 }
 201
 202 /* Wait until register `reg', masked with `mask', becomes `cmp' */
 203 static int imgu_hw_wait(void __iomem *base, int reg, u32 mask, u32 cmp)
 204 {
 205         u32 val;
 206
 207         return readl_poll_timeout(base + reg, val, (val & mask) == cmp,
 208                                   1000, 100 * 1000);
 209 }
 210
 211 /* Initialize the IPU3 CSS hardware and associated h/w blocks */
 212
 213 int imgu_css_set_powerup(struct device *dev, void __iomem *base)
 214 {
 215         static const unsigned int freq = 450;
 216         u32 pm_ctrl, state, val;
 217
 218         dev_dbg(dev, "%s\n", __func__);
 219         /* Clear the CSS busy signal */
 220         readl(base + IMGU_REG_GP_BUSY);
 221         writel(0, base + IMGU_REG_GP_BUSY);
 222
 223         /* Wait for idle signal */
 224         if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
 225                          IMGU_STATE_IDLE_STS)) {
 226                 dev_err(dev, "failed to set CSS idle\n");
 227                 goto fail;
 228         }
 229
 230         /* Reset the css */
 231         writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
 232                base + IMGU_REG_PM_CTRL);
 233
 234         usleep_range(200, 300);
 235
 236         /** Prepare CSS */
 237
 238         pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
 239         state = readl(base + IMGU_REG_STATE);
 240
 241         dev_dbg(dev, "CSS pm_ctrl 0x%x state 0x%x (power %s)\n",
 242                 pm_ctrl, state, state & IMGU_STATE_POWER_DOWN ? "down" : "up");
 243
 244         /* Power up CSS using wrapper */
 245         if (state & IMGU_STATE_POWER_DOWN) {
 246                 writel(IMGU_PM_CTRL_RACE_TO_HALT | IMGU_PM_CTRL_START,
 247                        base + IMGU_REG_PM_CTRL);
 248                 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL,
 249                                  IMGU_PM_CTRL_START, 0)) {
 250                         dev_err(dev, "failed to power up CSS\n");
 251                         goto fail;
 252                 }
 253                 usleep_range(2000, 3000);
 254         } else {
 255                 writel(IMGU_PM_CTRL_RACE_TO_HALT, base + IMGU_REG_PM_CTRL);
 256         }
 257
 258         /* Set the busy bit */
 259         writel(readl(base + IMGU_REG_GP_BUSY) | 1, base + IMGU_REG_GP_BUSY);
 260
 261         /* Set CSS clock frequency */
 262         pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
 263         val = pm_ctrl & ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
 264         writel(val, base + IMGU_REG_PM_CTRL);
 265         writel(0, base + IMGU_REG_GP_BUSY);
 266         if (imgu_hw_wait(base, IMGU_REG_STATE,
 267                          IMGU_STATE_PWRDNM_FSM_MASK, 0)) {
 268                 dev_err(dev, "failed to pwrdn CSS\n");
 269                 goto fail;
 270         }
 271         val = (freq / IMGU_SYSTEM_REQ_FREQ_DIVIDER) & IMGU_SYSTEM_REQ_FREQ_MASK;
 272         writel(val, base + IMGU_REG_SYSTEM_REQ);
 273         writel(1, base + IMGU_REG_GP_BUSY);
 274         writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_HALT,
 275                base + IMGU_REG_PM_CTRL);
 276         if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
 277                          IMGU_STATE_HALT_STS)) {
 278                 dev_err(dev, "failed to halt CSS\n");
 279                 goto fail;
 280         }
 281
 282         writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_START,
 283                base + IMGU_REG_PM_CTRL);
 284         if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, IMGU_PM_CTRL_START, 0)) {
 285                 dev_err(dev, "failed to start CSS\n");
 286                 goto fail;
 287         }
 288         writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_UNHALT,
 289                base + IMGU_REG_PM_CTRL);
 290
 291         val = readl(base + IMGU_REG_PM_CTRL);   /* get pm_ctrl */
 292         val &= ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
 293         val |= pm_ctrl & (IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
 294         writel(val, base + IMGU_REG_PM_CTRL);
 295
 296         return 0;
 297
 298 fail:
 299         imgu_css_set_powerdown(dev, base);
 300         return -EIO;
 301 }
 302
 303 void imgu_css_set_powerdown(struct device *dev, void __iomem *base)
 304 {
 305         dev_dbg(dev, "%s\n", __func__);
 306         /* wait for cio idle signal */
 307         if (imgu_hw_wait(base, IMGU_REG_CIO_GATE_BURST_STATE,
 308                          IMGU_CIO_GATE_BURST_MASK, 0))
 309                 dev_warn(dev, "wait cio gate idle timeout");
 310
 311         /* wait for css idle signal */
 312         if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
 313                          IMGU_STATE_IDLE_STS))
 314                 dev_warn(dev, "wait css idle timeout\n");
 315
 316         /* do halt-halted handshake with css */
 317         writel(1, base + IMGU_REG_GP_HALT);
 318         if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
 319                          IMGU_STATE_HALT_STS))
 320                 dev_warn(dev, "failed to halt css");
 321
 322         /* de-assert the busy bit */
 323         writel(0, base + IMGU_REG_GP_BUSY);
 324 }
 325
 326 static void imgu_css_hw_enable_irq(struct imgu_css *css)
 327 {
 328         void __iomem *const base = css->base;
 329         u32 val, i;
 330
 331         /* Set up interrupts */
 332
 333         /*
 334          * Enable IRQ on the SP which signals that SP goes to idle
 335          * (aka ready state) and set trigger to pulse
 336          */
 337         val = readl(base + IMGU_REG_SP_CTRL(0)) | IMGU_CTRL_IRQ_READY;
 338         writel(val, base + IMGU_REG_SP_CTRL(0));
 339         writel(val | IMGU_CTRL_IRQ_CLEAR, base + IMGU_REG_SP_CTRL(0));
 340
 341         /* Enable IRQs from the IMGU wrapper */
 342         writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_ENABLE);
 343         /* Clear */
 344         writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_STATUS);
 345
 346         /* Enable IRQs from main IRQ controller */
 347         writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(IMGU_IRQCTRL_MAIN));
 348         writel(0, base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
 349         writel(IMGU_IRQCTRL_IRQ_MASK,
 350                base + IMGU_REG_IRQCTRL_EDGE(IMGU_IRQCTRL_MAIN));
 351         writel(IMGU_IRQCTRL_IRQ_MASK,
 352                base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
 353         writel(IMGU_IRQCTRL_IRQ_MASK,
 354                base + IMGU_REG_IRQCTRL_CLEAR(IMGU_IRQCTRL_MAIN));
 355         writel(IMGU_IRQCTRL_IRQ_MASK,
 356                base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
 357         /* Wait for write complete */
 358         readl(base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
 359
 360         /* Enable IRQs from SP0 and SP1 controllers */
 361         for (i = IMGU_IRQCTRL_SP0; i <= IMGU_IRQCTRL_SP1; i++) {
 362                 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(i));
 363                 writel(0, base + IMGU_REG_IRQCTRL_MASK(i));
 364                 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_EDGE(i));
 365                 writel(IMGU_IRQCTRL_IRQ_MASK,
 366                        base + IMGU_REG_IRQCTRL_ENABLE(i));
 367                 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_CLEAR(i));
 368                 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_MASK(i));
 369                 /* Wait for write complete */
 370                 readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
 371         }
 372 }
 373
 374 static int imgu_css_hw_init(struct imgu_css *css)
 375 {
 376         /* For checking that streaming monitor statuses are valid */
 377         static const struct {
 378                 u32 reg;
 379                 u32 mask;
 380                 const char *name;
 381         } stream_monitors[] = {
 382                 {
 383                         IMGU_REG_GP_SP1_STRMON_STAT,
 384                         IMGU_GP_STRMON_STAT_ISP_PORT_SP12ISP,
 385                         "ISP0 to SP0"
 386                 }, {
 387                         IMGU_REG_GP_ISP_STRMON_STAT,
 388                         IMGU_GP_STRMON_STAT_SP1_PORT_ISP2SP1,
 389                         "SP0 to ISP0"
 390                 }, {
 391                         IMGU_REG_GP_MOD_STRMON_STAT,
 392                         IMGU_GP_STRMON_STAT_MOD_PORT_ISP2DMA,
 393                         "ISP0 to DMA0"
 394                 }, {
 395                         IMGU_REG_GP_ISP_STRMON_STAT,
 396                         IMGU_GP_STRMON_STAT_ISP_PORT_DMA2ISP,
 397                         "DMA0 to ISP0"
 398                 }, {
 399                         IMGU_REG_GP_MOD_STRMON_STAT,
 400                         IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
 401                         "ISP0 to GDC0"
 402                 }, {
 403                         IMGU_REG_GP_MOD_STRMON_STAT,
 404                         IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
 405                         "GDC0 to ISP0"
 406                 }, {
 407                         IMGU_REG_GP_MOD_STRMON_STAT,
 408                         IMGU_GP_STRMON_STAT_MOD_PORT_SP12DMA,
 409                         "SP0 to DMA0"
 410                 }, {
 411                         IMGU_REG_GP_SP1_STRMON_STAT,
 412                         IMGU_GP_STRMON_STAT_SP1_PORT_DMA2SP1,
 413                         "DMA0 to SP0"
 414                 }, {
 415                         IMGU_REG_GP_MOD_STRMON_STAT,
 416                         IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
 417                         "SP0 to GDC0"
 418                 }, {
 419                         IMGU_REG_GP_MOD_STRMON_STAT,
 420                         IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
 421                         "GDC0 to SP0"
 422                 },
 423         };
 424
 425         struct device *dev = css->dev;
 426         void __iomem *const base = css->base;
 427         u32 val, i;
 428
 429         /* Set instruction cache address and inv bit for ISP, SP, and SP1 */
 430         for (i = 0; i < IMGU_NUM_SP; i++) {
 431                 struct imgu_fw_info *bi =
 432                                         &css->fwp->binary_header[css->fw_sp[i]];
 433
 434                 writel(css->binary[css->fw_sp[i]].daddr,
 435                        base + IMGU_REG_SP_ICACHE_ADDR(bi->type));
 436                 writel(readl(base + IMGU_REG_SP_CTRL(bi->type)) |
 437                        IMGU_CTRL_ICACHE_INV,
 438                        base + IMGU_REG_SP_CTRL(bi->type));
 439         }
 440         writel(css->binary[css->fw_bl].daddr, base + IMGU_REG_ISP_ICACHE_ADDR);
 441         writel(readl(base + IMGU_REG_ISP_CTRL) | IMGU_CTRL_ICACHE_INV,
 442                base + IMGU_REG_ISP_CTRL);
 443
 444         /* Check that IMGU hardware is ready */
 445
 446         if (!(readl(base + IMGU_REG_SP_CTRL(0)) & IMGU_CTRL_IDLE)) {
 447                 dev_err(dev, "SP is not idle\n");
 448                 return -EIO;
 449         }
 450         if (!(readl(base + IMGU_REG_ISP_CTRL) & IMGU_CTRL_IDLE)) {
 451                 dev_err(dev, "ISP is not idle\n");
 452                 return -EIO;
 453         }
 454
 455         for (i = 0; i < ARRAY_SIZE(stream_monitors); i++) {
 456                 val = readl(base + stream_monitors[i].reg);
 457                 if (val & stream_monitors[i].mask) {
 458                         dev_err(dev, "error: Stream monitor %s is valid\n",
 459                                 stream_monitors[i].name);
 460                         return -EIO;
 461                 }
 462         }
 463
 464         /* Initialize GDC with default values */
 465
 466         for (i = 0; i < ARRAY_SIZE(imgu_css_gdc_lut[0]); i++) {
 467                 u32 val0 = imgu_css_gdc_lut[0][i] & IMGU_GDC_LUT_MASK;
 468                 u32 val1 = imgu_css_gdc_lut[1][i] & IMGU_GDC_LUT_MASK;
 469                 u32 val2 = imgu_css_gdc_lut[2][i] & IMGU_GDC_LUT_MASK;
 470                 u32 val3 = imgu_css_gdc_lut[3][i] & IMGU_GDC_LUT_MASK;
 471
 472                 writel(val0 | (val1 << 16),
 473                        base + IMGU_REG_GDC_LUT_BASE + i * 8);
 474                 writel(val2 | (val3 << 16),
 475                        base + IMGU_REG_GDC_LUT_BASE + i * 8 + 4);
 476         }
 477
 478         return 0;
 479 }
 480
 481 /* Boot the given IPU3 CSS SP */
 482 static int imgu_css_hw_start_sp(struct imgu_css *css, int sp)
 483 {
 484         void __iomem *const base = css->base;
 485         struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
 486         struct imgu_abi_sp_init_dmem_cfg dmem_cfg = {
 487                 .ddr_data_addr = css->binary[css->fw_sp[sp]].daddr
 488                         + bi->blob.data_source,
 489                 .dmem_data_addr = bi->blob.data_target,
 490                 .dmem_bss_addr = bi->blob.bss_target,
 491                 .data_size = bi->blob.data_size,
 492                 .bss_size = bi->blob.bss_size,
 493                 .sp_id = sp,
 494         };
 495
 496         writes(&dmem_cfg, sizeof(dmem_cfg), base +
 497                IMGU_REG_SP_DMEM_BASE(sp) + bi->info.sp.init_dmem_data);
 498
 499         writel(bi->info.sp.sp_entry, base + IMGU_REG_SP_START_ADDR(sp));
 500
 501         writel(readl(base + IMGU_REG_SP_CTRL(sp))
 502                 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_SP_CTRL(sp));
 503
 504         if (imgu_hw_wait(css->base, IMGU_REG_SP_DMEM_BASE(sp)
 505                          + bi->info.sp.sw_state,
 506                          ~0, IMGU_ABI_SP_SWSTATE_INITIALIZED))
 507                 return -EIO;
 508
 509         return 0;
 510 }
 511
 512 /* Start the IPU3 CSS ImgU (Imaging Unit) and all the SPs */
 513 static int imgu_css_hw_start(struct imgu_css *css)
 514 {
 515         static const u32 event_mask =
 516                 ((1 << IMGU_ABI_EVTTYPE_OUT_FRAME_DONE) |
 517                 (1 << IMGU_ABI_EVTTYPE_2ND_OUT_FRAME_DONE) |
 518                 (1 << IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE) |
 519                 (1 << IMGU_ABI_EVTTYPE_2ND_VF_OUT_FRAME_DONE) |
 520                 (1 << IMGU_ABI_EVTTYPE_3A_STATS_DONE) |
 521                 (1 << IMGU_ABI_EVTTYPE_DIS_STATS_DONE) |
 522                 (1 << IMGU_ABI_EVTTYPE_PIPELINE_DONE) |
 523                 (1 << IMGU_ABI_EVTTYPE_FRAME_TAGGED) |
 524                 (1 << IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE) |
 525                 (1 << IMGU_ABI_EVTTYPE_METADATA_DONE) |
 526                 (1 << IMGU_ABI_EVTTYPE_ACC_STAGE_COMPLETE))
 527                 << IMGU_ABI_SP_COMM_EVENT_IRQ_MASK_OR_SHIFT;
 528
 529         void __iomem *const base = css->base;
 530         struct imgu_fw_info *bi, *bl = &css->fwp->binary_header[css->fw_bl];
 531         unsigned int i;
 532
 533         writel(IMGU_TLB_INVALIDATE, base + IMGU_REG_TLB_INVALIDATE);
 534
 535         /* Start bootloader */
 536
 537         writel(IMGU_ABI_BL_SWSTATE_BUSY,
 538                base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.sw_state);
 539         writel(IMGU_NUM_SP,
 540                base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.num_dma_cmds);
 541
 542         for (i = 0; i < IMGU_NUM_SP; i++) {
 543                 int j = IMGU_NUM_SP - i - 1;    /* load sp1 first, then sp0 */
 544                 struct imgu_fw_info *sp =
 545                                         &css->fwp->binary_header[css->fw_sp[j]];
 546                 struct imgu_abi_bl_dma_cmd_entry dma_cmd = {
 547                         .src_addr = css->binary[css->fw_sp[j]].daddr
 548                                 + sp->blob.text_source,
 549                         .size = sp->blob.text_size,
 550                         .dst_type = IMGU_ABI_BL_DMACMD_TYPE_SP_PMEM,
 551                         .dst_addr = IMGU_SP_PMEM_BASE(j),
 552                 };
 553
 554                 writes(&dma_cmd, sizeof(dma_cmd),
 555                        base + IMGU_REG_ISP_DMEM_BASE + i * sizeof(dma_cmd) +
 556                        bl->info.bl.dma_cmd_list);
 557         }
 558
 559         writel(bl->info.bl.bl_entry, base + IMGU_REG_ISP_START_ADDR);
 560
 561         writel(readl(base + IMGU_REG_ISP_CTRL)
 562                 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_ISP_CTRL);
 563         if (imgu_hw_wait(css->base, IMGU_REG_ISP_DMEM_BASE
 564                          + bl->info.bl.sw_state, ~0,
 565                          IMGU_ABI_BL_SWSTATE_OK)) {
 566                 dev_err(css->dev, "failed to start bootloader\n");
 567                 return -EIO;
 568         }
 569
 570         /* Start ISP */
 571
 572         memset(css->xmem_sp_group_ptrs.vaddr, 0,
 573                sizeof(struct imgu_abi_sp_group));
 574
 575         bi = &css->fwp->binary_header[css->fw_sp[0]];
 576
 577         writel(css->xmem_sp_group_ptrs.daddr,
 578                base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.per_frame_data);
 579
 580         writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
 581                base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state);
 582         writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
 583
 584         if (imgu_css_hw_start_sp(css, 0))
 585                 return -EIO;
 586
 587         writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.isp_started);
 588         writel(0, base + IMGU_REG_SP_DMEM_BASE(0) +
 589                 bi->info.sp.host_sp_queues_initialized);
 590         writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sleep_mode);
 591         writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
 592         writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(0)
 593                 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
 594
 595         /* Enable all events for all queues */
 596
 597         for (i = 0; i < IPU3_CSS_PIPE_ID_NUM; i++)
 598                 writel(event_mask, base + IMGU_REG_SP_DMEM_BASE(0)
 599                         + bi->info.sp.host_sp_com
 600                         + IMGU_ABI_SP_COMM_EVENT_IRQ_MASK(i));
 601         writel(1, base + IMGU_REG_SP_DMEM_BASE(0) +
 602                 bi->info.sp.host_sp_queues_initialized);
 603
 604         /* Start SP1 */
 605
 606         bi = &css->fwp->binary_header[css->fw_sp[1]];
 607
 608         writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
 609                base + IMGU_REG_SP_DMEM_BASE(1) + bi->info.sp.sw_state);
 610
 611         if (imgu_css_hw_start_sp(css, 1))
 612                 return -EIO;
 613
 614         writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(1)
 615                 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
 616
 617         return 0;
 618 }
 619
 620 static void imgu_css_hw_stop(struct imgu_css *css)
 621 {
 622         void __iomem *const base = css->base;
 623         struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
 624
 625         /* Stop fw */
 626         writel(IMGU_ABI_SP_COMM_COMMAND_TERMINATE,
 627                base + IMGU_REG_SP_DMEM_BASE(0) +
 628                bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
 629         if (imgu_hw_wait(css->base, IMGU_REG_SP_CTRL(0),
 630                          IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
 631                 dev_err(css->dev, "wait sp0 idle timeout.\n");
 632         if (readl(base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state) !=
 633                   IMGU_ABI_SP_SWSTATE_TERMINATED)
 634                 dev_err(css->dev, "sp0 is not terminated.\n");
 635         if (imgu_hw_wait(css->base, IMGU_REG_ISP_CTRL,
 636                          IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
 637                 dev_err(css->dev, "wait isp idle timeout\n");
 638 }
 639
 640 static void imgu_css_hw_cleanup(struct imgu_css *css)
 641 {
 642         void __iomem *const base = css->base;
 643
 644         /** Reset CSS **/
 645
 646         /* Clear the CSS busy signal */
 647         readl(base + IMGU_REG_GP_BUSY);
 648         writel(0, base + IMGU_REG_GP_BUSY);
 649
 650         /* Wait for idle signal */
 651         if (imgu_hw_wait(css->base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
 652                          IMGU_STATE_IDLE_STS))
 653                 dev_err(css->dev, "failed to shut down hw cleanly\n");
 654
 655         /* Reset the css */
 656         writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
 657                base + IMGU_REG_PM_CTRL);
 658
 659         usleep_range(200, 300);
 660 }
 661
 662 static void imgu_css_pipeline_cleanup(struct imgu_css *css, unsigned int pipe)
 663 {
 664         struct imgu_device *imgu = dev_get_drvdata(css->dev);
 665         unsigned int i;
 666
 667         imgu_css_pool_cleanup(imgu,
 668                               &css->pipes[pipe].pool.parameter_set_info);
 669         imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.acc);
 670         imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.gdc);
 671         imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.obgrid);
 672
 673         for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
 674                 imgu_css_pool_cleanup(imgu,
 675                                       &css->pipes[pipe].pool.binary_params_p[i]);
 676 }
 677
 678 /*
 679  * This function initializes various stages of the
 680  * IPU3 CSS ISP pipeline
 681  */
 682 static int imgu_css_pipeline_init(struct imgu_css *css, unsigned int pipe)
 683 {
 684         static const int BYPC = 2;      /* Bytes per component */
 685         static const struct imgu_abi_buffer_sp buffer_sp_init = {
 686                 .buf_src = {.queue_id = IMGU_ABI_QUEUE_EVENT_ID},
 687                 .buf_type = IMGU_ABI_BUFFER_TYPE_INVALID,
 688         };
 689
 690         struct imgu_abi_isp_iterator_config *cfg_iter;
 691         struct imgu_abi_isp_ref_config *cfg_ref;
 692         struct imgu_abi_isp_dvs_config *cfg_dvs;
 693         struct imgu_abi_isp_tnr3_config *cfg_tnr;
 694         struct imgu_abi_isp_ref_dmem_state *cfg_ref_state;
 695         struct imgu_abi_isp_tnr3_dmem_state *cfg_tnr_state;
 696
 697         const int stage = 0;
 698         unsigned int i, j;
 699
 700         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
 701         const struct imgu_fw_info *bi =
 702                         &css->fwp->binary_header[css_pipe->bindex];
 703         const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes;
 704
 705         struct imgu_fw_config_memory_offsets *cofs = (void *)css->fwp +
 706                 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_CONFIG];
 707         struct imgu_fw_state_memory_offsets *sofs = (void *)css->fwp +
 708                 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_STATE];
 709
 710         struct imgu_abi_isp_stage *isp_stage;
 711         struct imgu_abi_sp_stage *sp_stage;
 712         struct imgu_abi_sp_group *sp_group;
 713
 714         const unsigned int bds_width_pad =
 715                                 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
 716                                       2 * IPU3_UAPI_ISP_VEC_ELEMS);
 717
 718         const enum imgu_abi_memories m0 = IMGU_ABI_MEM_ISP_DMEM0;
 719         enum imgu_abi_param_class cfg = IMGU_ABI_PARAM_CLASS_CONFIG;
 720         void *vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
 721
 722         struct imgu_device *imgu = dev_get_drvdata(css->dev);
 723
 724         dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
 725
 726         /* Configure iterator */
 727
 728         cfg_iter = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 729                                                &cofs->dmem.iterator,
 730                                                sizeof(*cfg_iter), vaddr);
 731         if (!cfg_iter)
 732                 goto bad_firmware;
 733
 734         cfg_iter->input_info.res.width =
 735                                 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.width;
 736         cfg_iter->input_info.res.height =
 737                                 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.height;
 738         cfg_iter->input_info.padded_width =
 739                                 css_pipe->queue[IPU3_CSS_QUEUE_IN].width_pad;
 740         cfg_iter->input_info.format =
 741                         css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->frame_format;
 742         cfg_iter->input_info.raw_bit_depth =
 743                         css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bit_depth;
 744         cfg_iter->input_info.raw_bayer_order =
 745                         css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bayer_order;
 746         cfg_iter->input_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
 747
 748         cfg_iter->internal_info.res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
 749         cfg_iter->internal_info.res.height =
 750                                         css_pipe->rect[IPU3_CSS_RECT_BDS].height;
 751         cfg_iter->internal_info.padded_width = bds_width_pad;
 752         cfg_iter->internal_info.format =
 753                         css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
 754         cfg_iter->internal_info.raw_bit_depth =
 755                         css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth;
 756         cfg_iter->internal_info.raw_bayer_order =
 757                         css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order;
 758         cfg_iter->internal_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
 759
 760         cfg_iter->output_info.res.width =
 761                                 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.width;
 762         cfg_iter->output_info.res.height =
 763                                 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height;
 764         cfg_iter->output_info.padded_width =
 765                                 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
 766         cfg_iter->output_info.format =
 767                         css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
 768         cfg_iter->output_info.raw_bit_depth =
 769                         css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth;
 770         cfg_iter->output_info.raw_bayer_order =
 771                         css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order;
 772         cfg_iter->output_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
 773
 774         cfg_iter->vf_info.res.width =
 775                         css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.width;
 776         cfg_iter->vf_info.res.height =
 777                         css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height;
 778         cfg_iter->vf_info.padded_width =
 779                         css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
 780         cfg_iter->vf_info.format =
 781                         css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
 782         cfg_iter->vf_info.raw_bit_depth =
 783                         css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bit_depth;
 784         cfg_iter->vf_info.raw_bayer_order =
 785                         css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bayer_order;
 786         cfg_iter->vf_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
 787
 788         cfg_iter->dvs_envelope.width = css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
 789         cfg_iter->dvs_envelope.height =
 790                                 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
 791
 792         /* Configure reference (delay) frames */
 793
 794         cfg_ref = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 795                                               &cofs->dmem.ref,
 796                                               sizeof(*cfg_ref), vaddr);
 797         if (!cfg_ref)
 798                 goto bad_firmware;
 799
 800         cfg_ref->port_b.crop = 0;
 801         cfg_ref->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES / BYPC;
 802         cfg_ref->port_b.width =
 803                 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width;
 804         cfg_ref->port_b.stride =
 805                 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline;
 806         cfg_ref->width_a_over_b =
 807                                 IPU3_UAPI_ISP_VEC_ELEMS / cfg_ref->port_b.elems;
 808         cfg_ref->dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
 809         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) {
 810                 cfg_ref->ref_frame_addr_y[i] =
 811                         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr;
 812                 cfg_ref->ref_frame_addr_c[i] =
 813                         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr +
 814                         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline *
 815                         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
 816         }
 817         for (; i < IMGU_ABI_FRAMES_REF; i++) {
 818                 cfg_ref->ref_frame_addr_y[i] = 0;
 819                 cfg_ref->ref_frame_addr_c[i] = 0;
 820         }
 821
 822         /* Configure DVS (digital video stabilization) */
 823
 824         cfg_dvs = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 825                                               &cofs->dmem.dvs, sizeof(*cfg_dvs),
 826                                               vaddr);
 827         if (!cfg_dvs)
 828                 goto bad_firmware;
 829
 830         cfg_dvs->num_horizontal_blocks =
 831                         ALIGN(DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
 832                                            IMGU_DVS_BLOCK_W), 2);
 833         cfg_dvs->num_vertical_blocks =
 834                         DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
 835                                      IMGU_DVS_BLOCK_H);
 836
 837         /* Configure TNR (temporal noise reduction) */
 838
 839         if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
 840                 cfg_tnr = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 841                                                       &cofs->dmem.tnr3,
 842                                                       sizeof(*cfg_tnr),
 843                                                       vaddr);
 844                 if (!cfg_tnr)
 845                         goto bad_firmware;
 846
 847                 cfg_tnr->port_b.crop = 0;
 848                 cfg_tnr->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES;
 849                 cfg_tnr->port_b.width =
 850                         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
 851                 cfg_tnr->port_b.stride =
 852                         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline;
 853                 cfg_tnr->width_a_over_b =
 854                         IPU3_UAPI_ISP_VEC_ELEMS / cfg_tnr->port_b.elems;
 855                 cfg_tnr->frame_height =
 856                         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
 857                 cfg_tnr->delay_frame = IPU3_CSS_AUX_FRAMES - 1;
 858                 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
 859                         cfg_tnr->frame_addr[i] =
 860                                 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR]
 861                                         .mem[i].daddr;
 862                 for (; i < IMGU_ABI_FRAMES_TNR; i++)
 863                         cfg_tnr->frame_addr[i] = 0;
 864         }
 865
 866         /* Configure ref dmem state parameters */
 867
 868         cfg = IMGU_ABI_PARAM_CLASS_STATE;
 869         vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
 870
 871         cfg_ref_state = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 872                                                     &sofs->dmem.ref,
 873                                                     sizeof(*cfg_ref_state),
 874                                                     vaddr);
 875         if (!cfg_ref_state)
 876                 goto bad_firmware;
 877
 878         cfg_ref_state->ref_in_buf_idx = 0;
 879         cfg_ref_state->ref_out_buf_idx = 1;
 880
 881         /* Configure tnr dmem state parameters */
 882         if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
 883                 cfg_tnr_state =
 884                         imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 885                                                     &sofs->dmem.tnr3,
 886                                                     sizeof(*cfg_tnr_state),
 887                                                     vaddr);
 888                 if (!cfg_tnr_state)
 889                         goto bad_firmware;
 890
 891                 cfg_tnr_state->in_bufidx = 0;
 892                 cfg_tnr_state->out_bufidx = 1;
 893                 cfg_tnr_state->bypass_filter = 0;
 894                 cfg_tnr_state->total_frame_counter = 0;
 895                 for (i = 0; i < IMGU_ABI_BUF_SETS_TNR; i++)
 896                         cfg_tnr_state->buffer_frame_counter[i] = 0;
 897         }
 898
 899         /* Configure ISP stage */
 900
 901         isp_stage = css_pipe->xmem_isp_stage_ptrs[pipe][stage].vaddr;
 902         memset(isp_stage, 0, sizeof(*isp_stage));
 903         isp_stage->blob_info = bi->blob;
 904         isp_stage->binary_info = bi->info.isp.sp;
 905         strscpy(isp_stage->binary_name,
 906                 (char *)css->fwp + bi->blob.prog_name_offset,
 907                 sizeof(isp_stage->binary_name));
 908         isp_stage->mem_initializers = bi->info.isp.sp.mem_initializers;
 909         for (i = IMGU_ABI_PARAM_CLASS_CONFIG; i < IMGU_ABI_PARAM_CLASS_NUM; i++)
 910                 for (j = 0; j < IMGU_ABI_NUM_MEMORIES; j++)
 911                         isp_stage->mem_initializers.params[i][j].address =
 912                                         css_pipe->binary_params_cs[i - 1][j].daddr;
 913
 914         /* Configure SP stage */
 915
 916         sp_stage = css_pipe->xmem_sp_stage_ptrs[pipe][stage].vaddr;
 917         memset(sp_stage, 0, sizeof(*sp_stage));
 918
 919         sp_stage->frames.in.buf_attr = buffer_sp_init;
 920         for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++)
 921                 sp_stage->frames.out[i].buf_attr = buffer_sp_init;
 922         sp_stage->frames.out_vf.buf_attr = buffer_sp_init;
 923         sp_stage->frames.s3a_buf = buffer_sp_init;
 924         sp_stage->frames.dvs_buf = buffer_sp_init;
 925
 926         sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP;
 927         sp_stage->num = stage;
 928         sp_stage->isp_online = 0;
 929         sp_stage->isp_copy_vf = 0;
 930         sp_stage->isp_copy_output = 0;
 931
 932         sp_stage->enable.vf_output = css_pipe->vf_output_en;
 933
 934         sp_stage->frames.effective_in_res.width =
 935                                 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].width;
 936         sp_stage->frames.effective_in_res.height =
 937                                 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].height;
 938         sp_stage->frames.in.info.res.width =
 939                                 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.width;
 940         sp_stage->frames.in.info.res.height =
 941                                 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.height;
 942         sp_stage->frames.in.info.padded_width =
 943                                         css_pipe->queue[IPU3_CSS_QUEUE_IN].width_pad;
 944         sp_stage->frames.in.info.format =
 945                         css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->frame_format;
 946         sp_stage->frames.in.info.raw_bit_depth =
 947                         css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bit_depth;
 948         sp_stage->frames.in.info.raw_bayer_order =
 949                         css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bayer_order;
 950         sp_stage->frames.in.info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
 951         sp_stage->frames.in.buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID;
 952         sp_stage->frames.in.buf_attr.buf_type =
 953                                         IMGU_ABI_BUFFER_TYPE_INPUT_FRAME;
 954
 955         sp_stage->frames.out[0].info.res.width =
 956                                 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.width;
 957         sp_stage->frames.out[0].info.res.height =
 958                                 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height;
 959         sp_stage->frames.out[0].info.padded_width =
 960                                 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
 961         sp_stage->frames.out[0].info.format =
 962                         css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
 963         sp_stage->frames.out[0].info.raw_bit_depth =
 964                         css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth;
 965         sp_stage->frames.out[0].info.raw_bayer_order =
 966                         css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order;
 967         sp_stage->frames.out[0].info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
 968         sp_stage->frames.out[0].planes.nv.uv.offset =
 969                                 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad *
 970                                 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height;
 971         sp_stage->frames.out[0].buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID;
 972         sp_stage->frames.out[0].buf_attr.buf_type =
 973                                         IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME;
 974
 975         sp_stage->frames.out[1].buf_attr.buf_src.queue_id =
 976                                                         IMGU_ABI_QUEUE_EVENT_ID;
 977
 978         sp_stage->frames.internal_frame_info.res.width =
 979                                         css_pipe->rect[IPU3_CSS_RECT_BDS].width;
 980         sp_stage->frames.internal_frame_info.res.height =
 981                                         css_pipe->rect[IPU3_CSS_RECT_BDS].height;
 982         sp_stage->frames.internal_frame_info.padded_width = bds_width_pad;
 983
 984         sp_stage->frames.internal_frame_info.format =
 985                         css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
 986         sp_stage->frames.internal_frame_info.raw_bit_depth =
 987                         css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth;
 988         sp_stage->frames.internal_frame_info.raw_bayer_order =
 989                         css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order;
 990         sp_stage->frames.internal_frame_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
 991
 992         sp_stage->frames.out_vf.info.res.width =
 993                                 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.width;
 994         sp_stage->frames.out_vf.info.res.height =
 995                                 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height;
 996         sp_stage->frames.out_vf.info.padded_width =
 997                                         css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
 998         sp_stage->frames.out_vf.info.format =
 999                         css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
1000         sp_stage->frames.out_vf.info.raw_bit_depth =
1001                         css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bit_depth;
1002         sp_stage->frames.out_vf.info.raw_bayer_order =
1003                         css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bayer_order;
1004         sp_stage->frames.out_vf.info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
1005         sp_stage->frames.out_vf.planes.yuv.u.offset =
1006                                 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad *
1007                                 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height;
1008         sp_stage->frames.out_vf.planes.yuv.v.offset =
1009                         css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad *
1010                         css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height * 5 / 4;
1011         sp_stage->frames.out_vf.buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID;
1012         sp_stage->frames.out_vf.buf_attr.buf_type =
1013                                         IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME;
1014
1015         sp_stage->frames.s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID;
1016         sp_stage->frames.s3a_buf.buf_type = IMGU_ABI_BUFFER_TYPE_3A_STATISTICS;
1017
1018         sp_stage->frames.dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID;
1019         sp_stage->frames.dvs_buf.buf_type = IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS;
1020
1021         sp_stage->dvs_envelope.width = css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
1022         sp_stage->dvs_envelope.height =
1023                                 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
1024
1025         sp_stage->isp_pipe_version =
1026                                 bi->info.isp.sp.pipeline.isp_pipe_version;
1027         sp_stage->isp_deci_log_factor =
1028                         clamp(max(fls(css_pipe->rect[IPU3_CSS_RECT_BDS].width /
1029                                       IMGU_MAX_BQ_GRID_WIDTH),
1030                                   fls(css_pipe->rect[IPU3_CSS_RECT_BDS].height /
1031                                       IMGU_MAX_BQ_GRID_HEIGHT)) - 1, 3, 5);
1032         sp_stage->isp_vf_downscale_bits = 0;
1033         sp_stage->if_config_index = 255;
1034         sp_stage->sp_enable_xnr = 0;
1035         sp_stage->num_stripes = stripes;
1036         sp_stage->enable.s3a = 1;
1037         sp_stage->enable.dvs_stats = 0;
1038
1039         sp_stage->xmem_bin_addr = css->binary[css_pipe->bindex].daddr;
1040         sp_stage->xmem_map_addr = css_pipe->sp_ddr_ptrs.daddr;
1041         sp_stage->isp_stage_addr =
1042                 css_pipe->xmem_isp_stage_ptrs[pipe][stage].daddr;
1043
1044         /* Configure SP group */
1045
1046         sp_group = css->xmem_sp_group_ptrs.vaddr;
1047         memset(&sp_group->pipe[pipe], 0, sizeof(struct imgu_abi_sp_pipeline));
1048
1049         sp_group->pipe[pipe].num_stages = 1;
1050         sp_group->pipe[pipe].pipe_id = css_pipe->pipe_id;
1051         sp_group->pipe[pipe].thread_id = pipe;
1052         sp_group->pipe[pipe].pipe_num = pipe;
1053         sp_group->pipe[pipe].num_execs = -1;
1054         sp_group->pipe[pipe].pipe_qos_config = -1;
1055         sp_group->pipe[pipe].required_bds_factor = 0;
1056         sp_group->pipe[pipe].dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
1057         sp_group->pipe[pipe].inout_port_config =
1058                                         IMGU_ABI_PORT_CONFIG_TYPE_INPUT_HOST |
1059                                         IMGU_ABI_PORT_CONFIG_TYPE_OUTPUT_HOST;
1060         sp_group->pipe[pipe].scaler_pp_lut = 0;
1061         sp_group->pipe[pipe].shading.internal_frame_origin_x_bqs_on_sctbl = 0;
1062         sp_group->pipe[pipe].shading.internal_frame_origin_y_bqs_on_sctbl = 0;
1063         sp_group->pipe[pipe].sp_stage_addr[stage] =
1064                         css_pipe->xmem_sp_stage_ptrs[pipe][stage].daddr;
1065         sp_group->pipe[pipe].pipe_config =
1066                         bi->info.isp.sp.enable.params ? (1 << pipe) : 0;
1067         sp_group->pipe[pipe].pipe_config |= IMGU_ABI_PIPE_CONFIG_ACQUIRE_ISP;
1068
1069         /* Initialize parameter pools */
1070
1071         if (imgu_css_pool_init(imgu, &css_pipe->pool.parameter_set_info,
1072                                sizeof(struct imgu_abi_parameter_set_info)) ||
1073             imgu_css_pool_init(imgu, &css_pipe->pool.acc,
1074                                sizeof(struct imgu_abi_acc_param)) ||
1075             imgu_css_pool_init(imgu, &css_pipe->pool.gdc,
1076                                sizeof(struct imgu_abi_gdc_warp_param) *
1077                                3 * cfg_dvs->num_horizontal_blocks / 2 *
1078                                cfg_dvs->num_vertical_blocks) ||
1079             imgu_css_pool_init(imgu, &css_pipe->pool.obgrid,
1080                                imgu_css_fw_obgrid_size(
1081                                &css->fwp->binary_header[css_pipe->bindex])))
1082                 goto out_of_memory;
1083
1084         for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1085                 if (imgu_css_pool_init(imgu,
1086                                        &css_pipe->pool.binary_params_p[i],
1087                                        bi->info.isp.sp.mem_initializers.params
1088                                        [IMGU_ABI_PARAM_CLASS_PARAM][i].size))
1089                         goto out_of_memory;
1090
1091         return 0;
1092
1093 bad_firmware:
1094         imgu_css_pipeline_cleanup(css, pipe);
1095         return -EPROTO;
1096
1097 out_of_memory:
1098         imgu_css_pipeline_cleanup(css, pipe);
1099         return -ENOMEM;
1100 }
1101
1102 static u8 imgu_css_queue_pos(struct imgu_css *css, int queue, int thread)
1103 {
1104         static const unsigned int sp;
1105         void __iomem *const base = css->base;
1106         struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1107         struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1108                                                 bi->info.sp.host_sp_queue;
1109
1110         return queue >= 0 ? readb(&q->host2sp_bufq_info[thread][queue].end) :
1111                             readb(&q->host2sp_evtq_info.end);
1112 }
1113
1114 /* Sent data to sp using given buffer queue, or if queue < 0, event queue. */
1115 static int imgu_css_queue_data(struct imgu_css *css,
1116                                int queue, int thread, u32 data)
1117 {
1118         static const unsigned int sp;
1119         void __iomem *const base = css->base;
1120         struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1121         struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1122                                                 bi->info.sp.host_sp_queue;
1123         u8 size, start, end, end2;
1124
1125         if (queue >= 0) {
1126                 size = readb(&q->host2sp_bufq_info[thread][queue].size);
1127                 start = readb(&q->host2sp_bufq_info[thread][queue].start);
1128                 end = readb(&q->host2sp_bufq_info[thread][queue].end);
1129         } else {
1130                 size = readb(&q->host2sp_evtq_info.size);
1131                 start = readb(&q->host2sp_evtq_info.start);
1132                 end = readb(&q->host2sp_evtq_info.end);
1133         }
1134
1135         if (size == 0)
1136                 return -EIO;
1137
1138         end2 = (end + 1) % size;
1139         if (end2 == start)
1140                 return -EBUSY;  /* Queue full */
1141
1142         if (queue >= 0) {
1143                 writel(data, &q->host2sp_bufq[thread][queue][end]);
1144                 writeb(end2, &q->host2sp_bufq_info[thread][queue].end);
1145         } else {
1146                 writel(data, &q->host2sp_evtq[end]);
1147                 writeb(end2, &q->host2sp_evtq_info.end);
1148         }
1149
1150         return 0;
1151 }
1152
1153 /* Receive data using given buffer queue, or if queue < 0, event queue. */
1154 static int imgu_css_dequeue_data(struct imgu_css *css, int queue, u32 *data)
1155 {
1156         static const unsigned int sp;
1157         void __iomem *const base = css->base;
1158         struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1159         struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1160                                                 bi->info.sp.host_sp_queue;
1161         u8 size, start, end, start2;
1162
1163         if (queue >= 0) {
1164                 size = readb(&q->sp2host_bufq_info[queue].size);
1165                 start = readb(&q->sp2host_bufq_info[queue].start);
1166                 end = readb(&q->sp2host_bufq_info[queue].end);
1167         } else {
1168                 size = readb(&q->sp2host_evtq_info.size);
1169                 start = readb(&q->sp2host_evtq_info.start);
1170                 end = readb(&q->sp2host_evtq_info.end);
1171         }
1172
1173         if (size == 0)
1174                 return -EIO;
1175
1176         if (end == start)
1177                 return -EBUSY;  /* Queue empty */
1178
1179         start2 = (start + 1) % size;
1180
1181         if (queue >= 0) {
1182                 *data = readl(&q->sp2host_bufq[queue][start]);
1183                 writeb(start2, &q->sp2host_bufq_info[queue].start);
1184         } else {
1185                 int r;
1186
1187                 *data = readl(&q->sp2host_evtq[start]);
1188                 writeb(start2, &q->sp2host_evtq_info.start);
1189
1190                 /* Acknowledge events dequeued from event queue */
1191                 r = imgu_css_queue_data(css, queue, 0,
1192                                         IMGU_ABI_EVENT_EVENT_DEQUEUED);
1193                 if (r < 0)
1194                         return r;
1195         }
1196
1197         return 0;
1198 }
1199
1200 /* Free binary-specific resources */
1201 static void imgu_css_binary_cleanup(struct imgu_css *css, unsigned int pipe)
1202 {
1203         struct imgu_device *imgu = dev_get_drvdata(css->dev);
1204         unsigned int i, j;
1205
1206         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1207
1208         for (j = 0; j < IMGU_ABI_PARAM_CLASS_NUM - 1; j++)
1209                 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1210                         imgu_dmamap_free(imgu,
1211                                          &css_pipe->binary_params_cs[j][i]);
1212
1213         j = IPU3_CSS_AUX_FRAME_REF;
1214         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1215                 imgu_dmamap_free(imgu,
1216                                  &css_pipe->aux_frames[j].mem[i]);
1217
1218         j = IPU3_CSS_AUX_FRAME_TNR;
1219         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1220                 imgu_dmamap_free(imgu,
1221                                  &css_pipe->aux_frames[j].mem[i]);
1222 }
1223
1224 static int imgu_css_binary_preallocate(struct imgu_css *css, unsigned int pipe)
1225 {
1226         struct imgu_device *imgu = dev_get_drvdata(css->dev);
1227         unsigned int i, j;
1228
1229         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1230
1231         for (j = IMGU_ABI_PARAM_CLASS_CONFIG;
1232              j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1233                 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1234                         if (!imgu_dmamap_alloc(imgu,
1235                                                &css_pipe->binary_params_cs[j - 1][i],
1236                                                CSS_ABI_SIZE))
1237                                 goto out_of_memory;
1238
1239         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1240                 if (!imgu_dmamap_alloc(imgu,
1241                                        &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].
1242                                        mem[i], CSS_BDS_SIZE))
1243                         goto out_of_memory;
1244
1245         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1246                 if (!imgu_dmamap_alloc(imgu,
1247                                        &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].
1248                                        mem[i], CSS_GDC_SIZE))
1249                         goto out_of_memory;
1250
1251         return 0;
1252
1253 out_of_memory:
1254         imgu_css_binary_cleanup(css, pipe);
1255         return -ENOMEM;
1256 }
1257
1258 /* allocate binary-specific resources */
1259 static int imgu_css_binary_setup(struct imgu_css *css, unsigned int pipe)
1260 {
1261         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1262         struct imgu_fw_info *bi = &css->fwp->binary_header[css_pipe->bindex];
1263         struct imgu_device *imgu = dev_get_drvdata(css->dev);
1264         int i, j, size;
1265         static const int BYPC = 2;      /* Bytes per component */
1266         unsigned int w, h;
1267
1268         /* Allocate parameter memory blocks for this binary */
1269
1270         for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1271                 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) {
1272                         if (imgu_css_dma_buffer_resize(
1273                             imgu,
1274                             &css_pipe->binary_params_cs[j - 1][i],
1275                             bi->info.isp.sp.mem_initializers.params[j][i].size))
1276                                 goto out_of_memory;
1277                 }
1278
1279         /* Allocate internal frame buffers */
1280
1281         /* Reference frames for DVS, FRAME_FORMAT_YUV420_16 */
1282         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel = BYPC;
1283         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width =
1284                                         css_pipe->rect[IPU3_CSS_RECT_BDS].width;
1285         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height =
1286                                 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].height,
1287                                       IMGU_DVS_BLOCK_H) + 2 * IMGU_GDC_BUF_Y;
1288         h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
1289         w = ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
1290                   2 * IPU3_UAPI_ISP_VEC_ELEMS) + 2 * IMGU_GDC_BUF_X;
1291         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline =
1292                 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel * w;
1293         size = w * h * BYPC + (w / 2) * (h / 2) * BYPC * 2;
1294         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1295                 if (imgu_css_dma_buffer_resize(
1296                         imgu,
1297                         &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i],
1298                         size))
1299                         goto out_of_memory;
1300
1301         /* TNR frames for temporal noise reduction, FRAME_FORMAT_YUV_LINE */
1302         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperpixel = 1;
1303         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width =
1304                         roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
1305                                 bi->info.isp.sp.block.block_width *
1306                                 IPU3_UAPI_ISP_VEC_ELEMS);
1307         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height =
1308                         roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
1309                                 bi->info.isp.sp.block.output_block_height);
1310
1311         w = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
1312         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline = w;
1313         h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
1314         size = w * ALIGN(h * 3 / 2 + 3, 2);     /* +3 for vf_pp prefetch */
1315         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1316                 if (imgu_css_dma_buffer_resize(
1317                         imgu,
1318                         &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i],
1319                         size))
1320                         goto out_of_memory;
1321
1322         return 0;
1323
1324 out_of_memory:
1325         imgu_css_binary_cleanup(css, pipe);
1326         return -ENOMEM;
1327 }
1328
1329 int imgu_css_start_streaming(struct imgu_css *css)
1330 {
1331         u32 data;
1332         int r, pipe;
1333
1334         if (css->streaming)
1335                 return -EPROTO;
1336
1337         for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1338                 r = imgu_css_binary_setup(css, pipe);
1339                 if (r < 0)
1340                         return r;
1341         }
1342
1343         r = imgu_css_hw_init(css);
1344         if (r < 0)
1345                 return r;
1346
1347         r = imgu_css_hw_start(css);
1348         if (r < 0)
1349                 goto fail;
1350
1351         for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1352                 r = imgu_css_pipeline_init(css, pipe);
1353                 if (r < 0)
1354                         goto fail;
1355         }
1356
1357         css->streaming = true;
1358
1359         imgu_css_hw_enable_irq(css);
1360
1361         /* Initialize parameters to default */
1362         for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1363                 r = imgu_css_set_parameters(css, pipe, NULL);
1364                 if (r < 0)
1365                         goto fail;
1366         }
1367
1368         while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_A_ID, &data)))
1369                 ;
1370         if (r != -EBUSY)
1371                 goto fail;
1372
1373         while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_B_ID, &data)))
1374                 ;
1375         if (r != -EBUSY)
1376                 goto fail;
1377
1378         for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1379                 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1380                                         IMGU_ABI_EVENT_START_STREAM |
1381                                         pipe << 16);
1382                 if (r < 0)
1383                         goto fail;
1384         }
1385
1386         return 0;
1387
1388 fail:
1389         css->streaming = false;
1390         imgu_css_hw_cleanup(css);
1391         for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1392                 imgu_css_pipeline_cleanup(css, pipe);
1393                 imgu_css_binary_cleanup(css, pipe);
1394         }
1395
1396         return r;
1397 }
1398
1399 void imgu_css_stop_streaming(struct imgu_css *css)
1400 {
1401         struct imgu_css_buffer *b, *b0;
1402         int q, r, pipe;
1403
1404         for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1405                 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1406                                         IMGU_ABI_EVENT_STOP_STREAM);
1407                 if (r < 0)
1408                         dev_warn(css->dev, "failed on stop stream event\n");
1409         }
1410
1411         if (!css->streaming)
1412                 return;
1413
1414         imgu_css_hw_stop(css);
1415
1416         imgu_css_hw_cleanup(css);
1417
1418         for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1419                 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1420
1421                 imgu_css_pipeline_cleanup(css, pipe);
1422
1423                 spin_lock(&css_pipe->qlock);
1424                 for (q = 0; q < IPU3_CSS_QUEUES; q++)
1425                         list_for_each_entry_safe(b, b0,
1426                                                  &css_pipe->queue[q].bufs,
1427                                                  list) {
1428                                 b->state = IPU3_CSS_BUFFER_FAILED;
1429                                 list_del(&b->list);
1430                         }
1431                 spin_unlock(&css_pipe->qlock);
1432         }
1433
1434         css->streaming = false;
1435 }
1436
1437 bool imgu_css_pipe_queue_empty(struct imgu_css *css, unsigned int pipe)
1438 {
1439         int q;
1440         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1441
1442         spin_lock(&css_pipe->qlock);
1443         for (q = 0; q < IPU3_CSS_QUEUES; q++)
1444                 if (!list_empty(&css_pipe->queue[q].bufs))
1445                         break;
1446         spin_unlock(&css_pipe->qlock);
1447         return (q == IPU3_CSS_QUEUES);
1448 }
1449
1450 bool imgu_css_queue_empty(struct imgu_css *css)
1451 {
1452         unsigned int pipe;
1453         bool ret = 0;
1454
1455         for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1456                 ret &= imgu_css_pipe_queue_empty(css, pipe);
1457
1458         return ret;
1459 }
1460
1461 bool imgu_css_is_streaming(struct imgu_css *css)
1462 {
1463         return css->streaming;
1464 }
1465
1466 static int imgu_css_map_init(struct imgu_css *css, unsigned int pipe)
1467 {
1468         struct imgu_device *imgu = dev_get_drvdata(css->dev);
1469         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1470         unsigned int p, q, i;
1471
1472         /* Allocate and map common structures with imgu hardware */
1473         for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1474                 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1475                         if (!imgu_dmamap_alloc(imgu,
1476                                                &css_pipe->
1477                                                xmem_sp_stage_ptrs[p][i],
1478                                                sizeof(struct imgu_abi_sp_stage)))
1479                                 return -ENOMEM;
1480                         if (!imgu_dmamap_alloc(imgu,
1481                                                &css_pipe->
1482                                                xmem_isp_stage_ptrs[p][i],
1483                                                sizeof(struct imgu_abi_isp_stage)))
1484                                 return -ENOMEM;
1485                 }
1486
1487         if (!imgu_dmamap_alloc(imgu, &css_pipe->sp_ddr_ptrs,
1488                                ALIGN(sizeof(struct imgu_abi_ddr_address_map),
1489                                      IMGU_ABI_ISP_DDR_WORD_BYTES)))
1490                 return -ENOMEM;
1491
1492         for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1493                 unsigned int abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1494
1495                 for (i = 0; i < abi_buf_num; i++)
1496                         if (!imgu_dmamap_alloc(imgu,
1497                                                &css_pipe->abi_buffers[q][i],
1498                                                sizeof(struct imgu_abi_buffer)))
1499                                 return -ENOMEM;
1500         }
1501
1502         if (imgu_css_binary_preallocate(css, pipe)) {
1503                 imgu_css_binary_cleanup(css, pipe);
1504                 return -ENOMEM;
1505         }
1506
1507         return 0;
1508 }
1509
1510 static void imgu_css_pipe_cleanup(struct imgu_css *css, unsigned int pipe)
1511 {
1512         struct imgu_device *imgu = dev_get_drvdata(css->dev);
1513         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1514         unsigned int p, q, i, abi_buf_num;
1515
1516         imgu_css_binary_cleanup(css, pipe);
1517
1518         for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1519                 abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1520                 for (i = 0; i < abi_buf_num; i++)
1521                         imgu_dmamap_free(imgu, &css_pipe->abi_buffers[q][i]);
1522         }
1523
1524         for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1525                 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1526                         imgu_dmamap_free(imgu,
1527                                          &css_pipe->xmem_sp_stage_ptrs[p][i]);
1528                         imgu_dmamap_free(imgu,
1529                                          &css_pipe->xmem_isp_stage_ptrs[p][i]);
1530                 }
1531
1532         imgu_dmamap_free(imgu, &css_pipe->sp_ddr_ptrs);
1533 }
1534
1535 void imgu_css_cleanup(struct imgu_css *css)
1536 {
1537         struct imgu_device *imgu = dev_get_drvdata(css->dev);
1538         unsigned int pipe;
1539
1540         imgu_css_stop_streaming(css);
1541         for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1542                 imgu_css_pipe_cleanup(css, pipe);
1543         imgu_dmamap_free(imgu, &css->xmem_sp_group_ptrs);
1544         imgu_css_fw_cleanup(css);
1545 }
1546
1547 int imgu_css_init(struct device *dev, struct imgu_css *css,
1548                   void __iomem *base, int length)
1549 {
1550         struct imgu_device *imgu = dev_get_drvdata(dev);
1551         int r, q, pipe;
1552
1553         /* Initialize main data structure */
1554         css->dev = dev;
1555         css->base = base;
1556         css->iomem_length = length;
1557
1558         for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) {
1559                 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1560
1561                 css_pipe->vf_output_en = false;
1562                 spin_lock_init(&css_pipe->qlock);
1563                 css_pipe->bindex = IPU3_CSS_DEFAULT_BINARY;
1564                 css_pipe->pipe_id = IPU3_CSS_PIPE_ID_VIDEO;
1565                 for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1566                         r = imgu_css_queue_init(&css_pipe->queue[q], NULL, 0);
1567                         if (r)
1568                                 return r;
1569                 }
1570                 r = imgu_css_map_init(css, pipe);
1571                 if (r) {
1572                         imgu_css_cleanup(css);
1573                         return r;
1574                 }
1575         }
1576         if (!imgu_dmamap_alloc(imgu, &css->xmem_sp_group_ptrs,
1577                                sizeof(struct imgu_abi_sp_group)))
1578                 return -ENOMEM;
1579
1580         r = imgu_css_fw_init(css);
1581         if (r)
1582                 return r;
1583
1584         return 0;
1585 }
1586
1587 static u32 imgu_css_adjust(u32 res, u32 align)
1588 {
1589         u32 val = max_t(u32, IPU3_CSS_MIN_RES, res);
1590
1591         return DIV_ROUND_CLOSEST(val, align) * align;
1592 }
1593
1594 /* Select a binary matching the required resolutions and formats */
1595 static int imgu_css_find_binary(struct imgu_css *css,
1596                                 unsigned int pipe,
1597                                 struct imgu_css_queue queue[IPU3_CSS_QUEUES],
1598                                 struct v4l2_rect rects[IPU3_CSS_RECTS])
1599 {
1600         const int binary_nr = css->fwp->file_header.binary_nr;
1601         unsigned int binary_mode =
1602                 (css->pipes[pipe].pipe_id == IPU3_CSS_PIPE_ID_CAPTURE) ?
1603                 IA_CSS_BINARY_MODE_PRIMARY : IA_CSS_BINARY_MODE_VIDEO;
1604         const struct v4l2_pix_format_mplane *in =
1605                                         &queue[IPU3_CSS_QUEUE_IN].fmt.mpix;
1606         const struct v4l2_pix_format_mplane *out =
1607                                         &queue[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1608         const struct v4l2_pix_format_mplane *vf =
1609                                         &queue[IPU3_CSS_QUEUE_VF].fmt.mpix;
1610         u32 stripe_w = 0, stripe_h = 0;
1611         const char *name;
1612         int i, j;
1613
1614         if (!imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_IN]))
1615                 return -EINVAL;
1616
1617         /* Find out the strip size boundary */
1618         for (i = 0; i < binary_nr; i++) {
1619                 struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1620
1621                 u32 max_width = bi->info.isp.sp.output.max_width;
1622                 u32 max_height = bi->info.isp.sp.output.max_height;
1623
1624                 if (bi->info.isp.sp.iterator.num_stripes <= 1) {
1625                         stripe_w = stripe_w ?
1626                                 min(stripe_w, max_width) : max_width;
1627                         stripe_h = stripe_h ?
1628                                 min(stripe_h, max_height) : max_height;
1629                 }
1630         }
1631
1632         for (i = 0; i < binary_nr; i++) {
1633                 struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1634                 enum imgu_abi_frame_format q_fmt;
1635
1636                 name = (void *)css->fwp + bi->blob.prog_name_offset;
1637
1638                 /* Check that binary supports memory-to-memory processing */
1639                 if (bi->info.isp.sp.input.source !=
1640                     IMGU_ABI_BINARY_INPUT_SOURCE_MEMORY)
1641                         continue;
1642
1643                 /* Check that binary supports raw10 input */
1644                 if (!bi->info.isp.sp.enable.input_feeder &&
1645                     !bi->info.isp.sp.enable.input_raw)
1646                         continue;
1647
1648                 /* Check binary mode */
1649                 if (bi->info.isp.sp.pipeline.mode != binary_mode)
1650                         continue;
1651
1652                 /* Since input is RGGB bayer, need to process colors */
1653                 if (bi->info.isp.sp.enable.luma_only)
1654                         continue;
1655
1656                 if (in->width < bi->info.isp.sp.input.min_width ||
1657                     in->width > bi->info.isp.sp.input.max_width ||
1658                     in->height < bi->info.isp.sp.input.min_height ||
1659                     in->height > bi->info.isp.sp.input.max_height)
1660                         continue;
1661
1662                 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_OUT])) {
1663                         if (bi->info.isp.num_output_pins <= 0)
1664                                 continue;
1665
1666                         q_fmt = queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
1667                         for (j = 0; j < bi->info.isp.num_output_formats; j++)
1668                                 if (bi->info.isp.output_formats[j] == q_fmt)
1669                                         break;
1670                         if (j >= bi->info.isp.num_output_formats)
1671                                 continue;
1672
1673                         if (out->width < bi->info.isp.sp.output.min_width ||
1674                             out->width > bi->info.isp.sp.output.max_width ||
1675                             out->height < bi->info.isp.sp.output.min_height ||
1676                             out->height > bi->info.isp.sp.output.max_height)
1677                                 continue;
1678
1679                         if (out->width > bi->info.isp.sp.internal.max_width ||
1680                             out->height > bi->info.isp.sp.internal.max_height)
1681                                 continue;
1682                 }
1683
1684                 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_VF])) {
1685                         if (bi->info.isp.num_output_pins <= 1)
1686                                 continue;
1687
1688                         q_fmt = queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
1689                         for (j = 0; j < bi->info.isp.num_output_formats; j++)
1690                                 if (bi->info.isp.output_formats[j] == q_fmt)
1691                                         break;
1692                         if (j >= bi->info.isp.num_output_formats)
1693                                 continue;
1694
1695                         if (vf->width < bi->info.isp.sp.output.min_width ||
1696                             vf->width > bi->info.isp.sp.output.max_width ||
1697                             vf->height < bi->info.isp.sp.output.min_height ||
1698                             vf->height > bi->info.isp.sp.output.max_height)
1699                                 continue;
1700                 }
1701
1702                 /* All checks passed, select the binary */
1703                 dev_dbg(css->dev, "using binary %s id = %u\n", name,
1704                         bi->info.isp.sp.id);
1705                 return i;
1706         }
1707
1708         /* Can not find suitable binary for these parameters */
1709         return -EINVAL;
1710 }
1711
1712 /*
1713  * Check that there is a binary matching requirements. Parameters may be
1714  * NULL indicating disabled input/output. Return negative if given
1715  * parameters can not be supported or on error, zero or positive indicating
1716  * found binary number. May modify the given parameters if not exact match
1717  * is found.
1718  */
1719 int imgu_css_fmt_try(struct imgu_css *css,
1720                      struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1721                      struct v4l2_rect *rects[IPU3_CSS_RECTS],
1722                      unsigned int pipe)
1723 {
1724         static const u32 EFF_ALIGN_W = 2;
1725         static const u32 BDS_ALIGN_W = 4;
1726         static const u32 OUT_ALIGN_W = 8;
1727         static const u32 OUT_ALIGN_H = 4;
1728         static const u32 VF_ALIGN_W  = 2;
1729         static const char *qnames[IPU3_CSS_QUEUES] = {
1730                 [IPU3_CSS_QUEUE_IN] = "in",
1731                 [IPU3_CSS_QUEUE_PARAMS]    = "params",
1732                 [IPU3_CSS_QUEUE_OUT] = "out",
1733                 [IPU3_CSS_QUEUE_VF] = "vf",
1734                 [IPU3_CSS_QUEUE_STAT_3A]   = "3a",
1735         };
1736         static const char *rnames[IPU3_CSS_RECTS] = {
1737                 [IPU3_CSS_RECT_EFFECTIVE] = "effective resolution",
1738                 [IPU3_CSS_RECT_BDS]       = "bayer-domain scaled resolution",
1739                 [IPU3_CSS_RECT_ENVELOPE]  = "DVS envelope size",
1740                 [IPU3_CSS_RECT_GDC]  = "GDC output res",
1741         };
1742         struct v4l2_rect r[IPU3_CSS_RECTS] = { };
1743         struct v4l2_rect *const eff = &r[IPU3_CSS_RECT_EFFECTIVE];
1744         struct v4l2_rect *const bds = &r[IPU3_CSS_RECT_BDS];
1745         struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE];
1746         struct v4l2_rect *const gdc = &r[IPU3_CSS_RECT_GDC];
1747         struct imgu_css_queue *q;
1748         struct v4l2_pix_format_mplane *in, *out, *vf;
1749         int i, s, ret;
1750
1751         q = kcalloc(IPU3_CSS_QUEUES, sizeof(struct imgu_css_queue), GFP_KERNEL);
1752         if (!q)
1753                 return -ENOMEM;
1754
1755         in  = &q[IPU3_CSS_QUEUE_IN].fmt.mpix;
1756         out = &q[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1757         vf  = &q[IPU3_CSS_QUEUE_VF].fmt.mpix;
1758
1759         /* Adjust all formats, get statistics buffer sizes and formats */
1760         for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1761                 if (fmts[i])
1762                         dev_dbg(css->dev, "%s %s: (%i,%i) fmt 0x%x\n", __func__,
1763                                 qnames[i], fmts[i]->width, fmts[i]->height,
1764                                 fmts[i]->pixelformat);
1765                 else
1766                         dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1767                                 qnames[i]);
1768                 if (imgu_css_queue_init(&q[i], fmts[i],
1769                                         IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1770                         dev_notice(css->dev, "can not initialize queue %s\n",
1771                                    qnames[i]);
1772                         ret = -EINVAL;
1773                         goto out;
1774                 }
1775         }
1776         for (i = 0; i < IPU3_CSS_RECTS; i++) {
1777                 if (rects[i]) {
1778                         dev_dbg(css->dev, "%s %s: (%i,%i)\n", __func__,
1779                                 rnames[i], rects[i]->width, rects[i]->height);
1780                         r[i].width  = rects[i]->width;
1781                         r[i].height = rects[i]->height;
1782                 } else {
1783                         dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1784                                 rnames[i]);
1785                 }
1786                 /* For now, force known good resolutions */
1787                 r[i].left = 0;
1788                 r[i].top  = 0;
1789         }
1790
1791         /* Always require one input and vf only if out is also enabled */
1792         if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_IN]) ||
1793             !imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1794                 dev_warn(css->dev, "required queues are disabled\n");
1795                 ret = -EINVAL;
1796                 goto out;
1797         }
1798
1799         if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1800                 out->width = in->width;
1801                 out->height = in->height;
1802         }
1803         if (eff->width <= 0 || eff->height <= 0) {
1804                 eff->width = in->width;
1805                 eff->height = in->height;
1806         }
1807         if (bds->width <= 0 || bds->height <= 0) {
1808                 bds->width = out->width;
1809                 bds->height = out->height;
1810         }
1811         if (gdc->width <= 0 || gdc->height <= 0) {
1812                 gdc->width = out->width;
1813                 gdc->height = out->height;
1814         }
1815
1816         in->width   = imgu_css_adjust(in->width, 1);
1817         in->height  = imgu_css_adjust(in->height, 1);
1818         eff->width  = imgu_css_adjust(eff->width, EFF_ALIGN_W);
1819         eff->height = imgu_css_adjust(eff->height, 1);
1820         bds->width  = imgu_css_adjust(bds->width, BDS_ALIGN_W);
1821         bds->height = imgu_css_adjust(bds->height, 1);
1822         gdc->width  = imgu_css_adjust(gdc->width, OUT_ALIGN_W);
1823         gdc->height = imgu_css_adjust(gdc->height, OUT_ALIGN_H);
1824         out->width  = imgu_css_adjust(out->width, OUT_ALIGN_W);
1825         out->height = imgu_css_adjust(out->height, OUT_ALIGN_H);
1826         vf->width   = imgu_css_adjust(vf->width, VF_ALIGN_W);
1827         vf->height  = imgu_css_adjust(vf->height, 1);
1828
1829         s = (bds->width - gdc->width) / 2;
1830         env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1831         s = (bds->height - gdc->height) / 2;
1832         env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1833
1834         ret = imgu_css_find_binary(css, pipe, q, r);
1835         if (ret < 0) {
1836                 dev_err(css->dev, "failed to find suitable binary\n");
1837                 ret = -EINVAL;
1838                 goto out;
1839         }
1840         css->pipes[pipe].bindex = ret;
1841
1842         dev_dbg(css->dev, "Binary index %d for pipe %d found.",
1843                 css->pipes[pipe].bindex, pipe);
1844
1845         /* Final adjustment and set back the queried formats */
1846         for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1847                 if (fmts[i]) {
1848                         if (imgu_css_queue_init(&q[i], &q[i].fmt.mpix,
1849                                                 IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1850                                 dev_err(css->dev,
1851                                         "final resolution adjustment failed\n");
1852                                 ret = -EINVAL;
1853                                 goto out;
1854                         }
1855                         *fmts[i] = q[i].fmt.mpix;
1856                 }
1857         }
1858
1859         for (i = 0; i < IPU3_CSS_RECTS; i++)
1860                 if (rects[i])
1861                         *rects[i] = r[i];
1862
1863         dev_dbg(css->dev,
1864                 "in(%u,%u) if(%u,%u) ds(%u,%u) gdc(%u,%u) out(%u,%u) vf(%u,%u)",
1865                  in->width, in->height, eff->width, eff->height,
1866                  bds->width, bds->height, gdc->width, gdc->height,
1867                  out->width, out->height, vf->width, vf->height);
1868
1869         ret = 0;
1870 out:
1871         kfree(q);
1872         return ret;
1873 }
1874
1875 int imgu_css_fmt_set(struct imgu_css *css,
1876                      struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1877                      struct v4l2_rect *rects[IPU3_CSS_RECTS],
1878                      unsigned int pipe)
1879 {
1880         struct v4l2_rect rect_data[IPU3_CSS_RECTS];
1881         struct v4l2_rect *all_rects[IPU3_CSS_RECTS];
1882         int i, r;
1883         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1884
1885         for (i = 0; i < IPU3_CSS_RECTS; i++) {
1886                 if (rects[i])
1887                         rect_data[i] = *rects[i];
1888                 else
1889                         memset(&rect_data[i], 0, sizeof(rect_data[i]));
1890                 all_rects[i] = &rect_data[i];
1891         }
1892         r = imgu_css_fmt_try(css, fmts, all_rects, pipe);
1893         if (r < 0)
1894                 return r;
1895
1896         for (i = 0; i < IPU3_CSS_QUEUES; i++)
1897                 if (imgu_css_queue_init(&css_pipe->queue[i], fmts[i],
1898                                         IPU3_CSS_QUEUE_TO_FLAGS(i)))
1899                         return -EINVAL;
1900         for (i = 0; i < IPU3_CSS_RECTS; i++) {
1901                 css_pipe->rect[i] = rect_data[i];
1902                 if (rects[i])
1903                         *rects[i] = rect_data[i];
1904         }
1905
1906         return 0;
1907 }
1908
1909 int imgu_css_meta_fmt_set(struct v4l2_meta_format *fmt)
1910 {
1911         switch (fmt->dataformat) {
1912         case V4L2_META_FMT_IPU3_PARAMS:
1913                 fmt->buffersize = sizeof(struct ipu3_uapi_params);
1914                 break;
1915         case V4L2_META_FMT_IPU3_STAT_3A:
1916                 fmt->buffersize = sizeof(struct ipu3_uapi_stats_3a);
1917                 break;
1918         default:
1919                 return -EINVAL;
1920         }
1921
1922         return 0;
1923 }
1924
1925 /*
1926  * Queue given buffer to CSS. imgu_css_buf_prepare() must have been first
1927  * called for the buffer. May be called from interrupt context.
1928  * Returns 0 on success, -EBUSY if the buffer queue is full, or some other
1929  * code on error conditions.
1930  */
1931 int imgu_css_buf_queue(struct imgu_css *css, unsigned int pipe,
1932                        struct imgu_css_buffer *b)
1933 {
1934         struct imgu_abi_buffer *abi_buf;
1935         struct imgu_addr_t *buf_addr;
1936         u32 data;
1937         int r;
1938         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1939
1940         if (!css->streaming)
1941                 return -EPROTO; /* CSS or buffer in wrong state */
1942
1943         if (b->queue >= IPU3_CSS_QUEUES || !imgu_css_queues[b->queue].qid)
1944                 return -EINVAL;
1945
1946         b->queue_pos = imgu_css_queue_pos(css, imgu_css_queues[b->queue].qid,
1947                                           pipe);
1948
1949         if (b->queue_pos >= ARRAY_SIZE(css->pipes[pipe].abi_buffers[b->queue]))
1950                 return -EIO;
1951         abi_buf = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].vaddr;
1952
1953         /* Fill struct abi_buffer for firmware */
1954         memset(abi_buf, 0, sizeof(*abi_buf));
1955
1956         buf_addr = (void *)abi_buf + imgu_css_queues[b->queue].ptr_ofs;
1957         *(imgu_addr_t *)buf_addr = b->daddr;
1958
1959         if (b->queue == IPU3_CSS_QUEUE_STAT_3A)
1960                 abi_buf->payload.s3a.data.dmem.s3a_tbl = b->daddr;
1961
1962         if (b->queue == IPU3_CSS_QUEUE_OUT)
1963                 abi_buf->payload.frame.padded_width =
1964                                 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
1965
1966         if (b->queue == IPU3_CSS_QUEUE_VF)
1967                 abi_buf->payload.frame.padded_width =
1968                                         css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
1969
1970         spin_lock(&css_pipe->qlock);
1971         list_add_tail(&b->list, &css_pipe->queue[b->queue].bufs);
1972         spin_unlock(&css_pipe->qlock);
1973         b->state = IPU3_CSS_BUFFER_QUEUED;
1974
1975         data = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].daddr;
1976         r = imgu_css_queue_data(css, imgu_css_queues[b->queue].qid,
1977                                 pipe, data);
1978         if (r < 0)
1979                 goto queueing_failed;
1980
1981         data = IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
1982                                               imgu_css_queues[b->queue].qid);
1983         r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, data);
1984         if (r < 0)
1985                 goto queueing_failed;
1986
1987         dev_dbg(css->dev, "queued buffer %p to css queue %i in pipe %d\n",
1988                 b, b->queue, pipe);
1989
1990         return 0;
1991
1992 queueing_failed:
1993         b->state = (r == -EBUSY || r == -EAGAIN) ?
1994                 IPU3_CSS_BUFFER_NEW : IPU3_CSS_BUFFER_FAILED;
1995         list_del(&b->list);
1996
1997         return r;
1998 }
1999
2000 /*
2001  * Get next ready CSS buffer. Returns -EAGAIN in which case the function
2002  * should be called again, or -EBUSY which means that there are no more
2003  * buffers available. May be called from interrupt context.
2004  */
2005 struct imgu_css_buffer *imgu_css_buf_dequeue(struct imgu_css *css)
2006 {
2007         static const unsigned char evtype_to_queue[] = {
2008                 [IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE] = IPU3_CSS_QUEUE_IN,
2009                 [IMGU_ABI_EVTTYPE_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_OUT,
2010                 [IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_VF,
2011                 [IMGU_ABI_EVTTYPE_3A_STATS_DONE] = IPU3_CSS_QUEUE_STAT_3A,
2012         };
2013         struct imgu_css_buffer *b = ERR_PTR(-EAGAIN);
2014         u32 event, daddr;
2015         int evtype, pipe, pipeid, queue, qid, r;
2016         struct imgu_css_pipe *css_pipe;
2017
2018         if (!css->streaming)
2019                 return ERR_PTR(-EPROTO);
2020
2021         r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
2022         if (r < 0)
2023                 return ERR_PTR(r);
2024
2025         evtype = (event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
2026                   IMGU_ABI_EVTTYPE_EVENT_SHIFT;
2027
2028         switch (evtype) {
2029         case IMGU_ABI_EVTTYPE_OUT_FRAME_DONE:
2030         case IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE:
2031         case IMGU_ABI_EVTTYPE_3A_STATS_DONE:
2032         case IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE:
2033                 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
2034                         IMGU_ABI_EVTTYPE_PIPE_SHIFT;
2035                 pipeid = (event & IMGU_ABI_EVTTYPE_PIPEID_MASK) >>
2036                         IMGU_ABI_EVTTYPE_PIPEID_SHIFT;
2037                 queue = evtype_to_queue[evtype];
2038                 qid = imgu_css_queues[queue].qid;
2039
2040                 if (pipe >= IMGU_MAX_PIPE_NUM) {
2041                         dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2042                         return ERR_PTR(-EIO);
2043                 }
2044
2045                 if (qid >= IMGU_ABI_QUEUE_NUM) {
2046                         dev_err(css->dev, "Invalid qid: %i\n", qid);
2047                         return ERR_PTR(-EIO);
2048                 }
2049                 css_pipe = &css->pipes[pipe];
2050                 dev_dbg(css->dev,
2051                         "event: buffer done 0x%x queue %i pipe %i pipeid %i\n",
2052                         event, queue, pipe, pipeid);
2053
2054                 r = imgu_css_dequeue_data(css, qid, &daddr);
2055                 if (r < 0) {
2056                         dev_err(css->dev, "failed to dequeue buffer\n");
2057                         /* Force real error, not -EBUSY */
2058                         return ERR_PTR(-EIO);
2059                 }
2060
2061                 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2062                                         IMGU_ABI_EVENT_BUFFER_DEQUEUED(qid));
2063                 if (r < 0) {
2064                         dev_err(css->dev, "failed to queue event\n");
2065                         return ERR_PTR(-EIO);
2066                 }
2067
2068                 spin_lock(&css_pipe->qlock);
2069                 if (list_empty(&css_pipe->queue[queue].bufs)) {
2070                         spin_unlock(&css_pipe->qlock);
2071                         dev_err(css->dev, "event on empty queue\n");
2072                         return ERR_PTR(-EIO);
2073                 }
2074                 b = list_first_entry(&css_pipe->queue[queue].bufs,
2075                                      struct imgu_css_buffer, list);
2076                 if (queue != b->queue ||
2077                     daddr != css_pipe->abi_buffers
2078                         [b->queue][b->queue_pos].daddr) {
2079                         spin_unlock(&css_pipe->qlock);
2080                         dev_err(css->dev, "dequeued bad buffer 0x%x\n", daddr);
2081                         return ERR_PTR(-EIO);
2082                 }
2083
2084                 dev_dbg(css->dev, "buffer 0x%8x done from pipe %d\n", daddr, pipe);
2085                 b->pipe = pipe;
2086                 b->state = IPU3_CSS_BUFFER_DONE;
2087                 list_del(&b->list);
2088                 spin_unlock(&css_pipe->qlock);
2089                 break;
2090         case IMGU_ABI_EVTTYPE_PIPELINE_DONE:
2091                 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
2092                         IMGU_ABI_EVTTYPE_PIPE_SHIFT;
2093                 if (pipe >= IMGU_MAX_PIPE_NUM) {
2094                         dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2095                         return ERR_PTR(-EIO);
2096                 }
2097
2098                 css_pipe = &css->pipes[pipe];
2099                 dev_dbg(css->dev, "event: pipeline done 0x%8x for pipe %d\n",
2100                         event, pipe);
2101                 break;
2102         case IMGU_ABI_EVTTYPE_TIMER:
2103                 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
2104                 if (r < 0)
2105                         return ERR_PTR(r);
2106
2107                 if ((event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
2108                     IMGU_ABI_EVTTYPE_EVENT_SHIFT == IMGU_ABI_EVTTYPE_TIMER)
2109                         dev_dbg(css->dev, "event: timer\n");
2110                 else
2111                         dev_warn(css->dev, "half of timer event missing\n");
2112                 break;
2113         case IMGU_ABI_EVTTYPE_FW_WARNING:
2114                 dev_warn(css->dev, "event: firmware warning 0x%x\n", event);
2115                 break;
2116         case IMGU_ABI_EVTTYPE_FW_ASSERT:
2117                 dev_err(css->dev,
2118                         "event: firmware assert 0x%x module_id %i line_no %i\n",
2119                         event,
2120                         (event & IMGU_ABI_EVTTYPE_MODULEID_MASK) >>
2121                         IMGU_ABI_EVTTYPE_MODULEID_SHIFT,
2122                         swab16((event & IMGU_ABI_EVTTYPE_LINENO_MASK) >>
2123                                IMGU_ABI_EVTTYPE_LINENO_SHIFT));
2124                 break;
2125         default:
2126                 dev_warn(css->dev, "received unknown event 0x%x\n", event);
2127         }
2128
2129         return b;
2130 }
2131
2132 /*
2133  * Get a new set of parameters from pool and initialize them based on
2134  * the parameters params, gdc, and obgrid. Any of these may be NULL,
2135  * in which case the previously set parameters are used.
2136  * If parameters haven't been set previously, initialize from scratch.
2137  *
2138  * Return index to css->parameter_set_info which has the newly created
2139  * parameters or negative value on error.
2140  */
2141 int imgu_css_set_parameters(struct imgu_css *css, unsigned int pipe,
2142                             struct ipu3_uapi_params *set_params)
2143 {
2144         static const unsigned int queue_id = IMGU_ABI_QUEUE_A_ID;
2145         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
2146         const int stage = 0;
2147         const struct imgu_fw_info *bi;
2148         int obgrid_size;
2149         unsigned int stripes, i;
2150         struct ipu3_uapi_flags *use = set_params ? &set_params->use : NULL;
2151
2152         /* Destination buffers which are filled here */
2153         struct imgu_abi_parameter_set_info *param_set;
2154         struct imgu_abi_acc_param *acc = NULL;
2155         struct imgu_abi_gdc_warp_param *gdc = NULL;
2156         struct ipu3_uapi_obgrid_param *obgrid = NULL;
2157         const struct imgu_css_map *map;
2158         void *vmem0 = NULL;
2159         void *dmem0 = NULL;
2160
2161         enum imgu_abi_memories m;
2162         int r = -EBUSY;
2163
2164         if (!css->streaming)
2165                 return -EPROTO;
2166
2167         dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
2168
2169         bi = &css->fwp->binary_header[css_pipe->bindex];
2170         obgrid_size = imgu_css_fw_obgrid_size(bi);
2171         stripes = bi->info.isp.sp.iterator.num_stripes ? : 1;
2172
2173         imgu_css_pool_get(&css_pipe->pool.parameter_set_info);
2174         param_set = imgu_css_pool_last(&css_pipe->pool.parameter_set_info,
2175                                        0)->vaddr;
2176
2177         /* Get a new acc only if new parameters given, or none yet */
2178         map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2179         if (set_params || !map->vaddr) {
2180                 imgu_css_pool_get(&css_pipe->pool.acc);
2181                 map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2182                 acc = map->vaddr;
2183         }
2184
2185         /* Get new VMEM0 only if needed, or none yet */
2186         m = IMGU_ABI_MEM_ISP_VMEM0;
2187         map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2188         if (!map->vaddr || (set_params && (set_params->use.lin_vmem_params ||
2189                                            set_params->use.tnr3_vmem_params ||
2190                                            set_params->use.xnr3_vmem_params))) {
2191                 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2192                 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2193                 vmem0 = map->vaddr;
2194         }
2195
2196         /* Get new DMEM0 only if needed, or none yet */
2197         m = IMGU_ABI_MEM_ISP_DMEM0;
2198         map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2199         if (!map->vaddr || (set_params && (set_params->use.tnr3_dmem_params ||
2200                                            set_params->use.xnr3_dmem_params))) {
2201                 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2202                 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2203                 dmem0 = map->vaddr;
2204         }
2205
2206         /* Configure acc parameter cluster */
2207         if (acc) {
2208                 /* get acc_old */
2209                 map = imgu_css_pool_last(&css_pipe->pool.acc, 1);
2210                 /* user acc */
2211                 r = imgu_css_cfg_acc(css, pipe, use, acc, map->vaddr,
2212                         set_params ? &set_params->acc_param : NULL);
2213                 if (r < 0)
2214                         goto fail;
2215         }
2216
2217         /* Configure late binding parameters */
2218         if (vmem0) {
2219                 m = IMGU_ABI_MEM_ISP_VMEM0;
2220                 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2221                 r = imgu_css_cfg_vmem0(css, pipe, use, vmem0,
2222                                        map->vaddr, set_params);
2223                 if (r < 0)
2224                         goto fail;
2225         }
2226
2227         if (dmem0) {
2228                 m = IMGU_ABI_MEM_ISP_DMEM0;
2229                 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2230                 r = imgu_css_cfg_dmem0(css, pipe, use, dmem0,
2231                                        map->vaddr, set_params);
2232                 if (r < 0)
2233                         goto fail;
2234         }
2235
2236         /* Get a new gdc only if a new gdc is given, or none yet */
2237         if (bi->info.isp.sp.enable.dvs_6axis) {
2238                 unsigned int a = IPU3_CSS_AUX_FRAME_REF;
2239                 unsigned int g = IPU3_CSS_RECT_GDC;
2240                 unsigned int e = IPU3_CSS_RECT_ENVELOPE;
2241
2242                 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2243                 if (!map->vaddr) {
2244                         imgu_css_pool_get(&css_pipe->pool.gdc);
2245                         map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2246                         gdc = map->vaddr;
2247                         imgu_css_cfg_gdc_table(map->vaddr,
2248                                 css_pipe->aux_frames[a].bytesperline /
2249                                 css_pipe->aux_frames[a].bytesperpixel,
2250                                 css_pipe->aux_frames[a].height,
2251                                 css_pipe->rect[g].width,
2252                                 css_pipe->rect[g].height,
2253                                 css_pipe->rect[e].width,
2254                                 css_pipe->rect[e].height);
2255                 }
2256         }
2257
2258         /* Get a new obgrid only if a new obgrid is given, or none yet */
2259         map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2260         if (!map->vaddr || (set_params && set_params->use.obgrid_param)) {
2261                 imgu_css_pool_get(&css_pipe->pool.obgrid);
2262                 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2263                 obgrid = map->vaddr;
2264
2265                 /* Configure optical black level grid (obgrid) */
2266                 if (set_params && set_params->use.obgrid_param)
2267                         for (i = 0; i < obgrid_size / sizeof(*obgrid); i++)
2268                                 obgrid[i] = set_params->obgrid_param;
2269                 else
2270                         memset(obgrid, 0, obgrid_size);
2271         }
2272
2273         /* Configure parameter set info, queued to `queue_id' */
2274
2275         memset(param_set, 0, sizeof(*param_set));
2276         map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2277         param_set->mem_map.acc_cluster_params_for_sp = map->daddr;
2278
2279         map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2280         param_set->mem_map.dvs_6axis_params_y = map->daddr;
2281
2282         for (i = 0; i < stripes; i++) {
2283                 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2284                 param_set->mem_map.obgrid_tbl[i] =
2285                         map->daddr + (obgrid_size / stripes) * i;
2286         }
2287
2288         for (m = 0; m < IMGU_ABI_NUM_MEMORIES; m++) {
2289                 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2290                 param_set->mem_map.isp_mem_param[stage][m] = map->daddr;
2291         }
2292
2293         /* Then queue the new parameter buffer */
2294         map = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 0);
2295         r = imgu_css_queue_data(css, queue_id, pipe, map->daddr);
2296         if (r < 0)
2297                 goto fail;
2298
2299         r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2300                                 IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
2301                                                                queue_id));
2302         if (r < 0)
2303                 goto fail_no_put;
2304
2305         /* Finally dequeue all old parameter buffers */
2306
2307         do {
2308                 u32 daddr;
2309
2310                 r = imgu_css_dequeue_data(css, queue_id, &daddr);
2311                 if (r == -EBUSY)
2312                         break;
2313                 if (r)
2314                         goto fail_no_put;
2315                 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2316                                         IMGU_ABI_EVENT_BUFFER_DEQUEUED
2317                                         (queue_id));
2318                 if (r < 0) {
2319                         dev_err(css->dev, "failed to queue parameter event\n");
2320                         goto fail_no_put;
2321                 }
2322         } while (1);
2323
2324         return 0;
2325
2326 fail:
2327         /*
2328          * A failure, most likely the parameter queue was full.
2329          * Return error but continue streaming. User can try submitting new
2330          * parameters again later.
2331          */
2332
2333         imgu_css_pool_put(&css_pipe->pool.parameter_set_info);
2334         if (acc)
2335                 imgu_css_pool_put(&css_pipe->pool.acc);
2336         if (gdc)
2337                 imgu_css_pool_put(&css_pipe->pool.gdc);
2338         if (obgrid)
2339                 imgu_css_pool_put(&css_pipe->pool.obgrid);
2340         if (vmem0)
2341                 imgu_css_pool_put(
2342                         &css_pipe->pool.binary_params_p
2343                         [IMGU_ABI_MEM_ISP_VMEM0]);
2344         if (dmem0)
2345                 imgu_css_pool_put(
2346                         &css_pipe->pool.binary_params_p
2347                         [IMGU_ABI_MEM_ISP_DMEM0]);
2348
2349 fail_no_put:
2350         return r;
2351 }
2352
2353 int imgu_css_irq_ack(struct imgu_css *css)
2354 {
2355         static const int NUM_SWIRQS = 3;
2356         struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
2357         void __iomem *const base = css->base;
2358         u32 irq_status[IMGU_IRQCTRL_NUM];
2359         int i;
2360
2361         u32 imgu_status = readl(base + IMGU_REG_INT_STATUS);
2362
2363         writel(imgu_status, base + IMGU_REG_INT_STATUS);
2364         for (i = 0; i < IMGU_IRQCTRL_NUM; i++)
2365                 irq_status[i] = readl(base + IMGU_REG_IRQCTRL_STATUS(i));
2366
2367         for (i = 0; i < NUM_SWIRQS; i++) {
2368                 if (irq_status[IMGU_IRQCTRL_SP0] & IMGU_IRQCTRL_IRQ_SW_PIN(i)) {
2369                         /* SP SW interrupt */
2370                         u32 cnt = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2371                                         bi->info.sp.output);
2372                         u32 val = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2373                                         bi->info.sp.output + 4 + 4 * i);
2374
2375                         dev_dbg(css->dev, "%s: swirq %i cnt %i val 0x%x\n",
2376                                 __func__, i, cnt, val);
2377                 }
2378         }
2379
2380         for (i = IMGU_IRQCTRL_NUM - 1; i >= 0; i--)
2381                 if (irq_status[i]) {
2382                         writel(irq_status[i], base + IMGU_REG_IRQCTRL_CLEAR(i));
2383                         /* Wait for write to complete */
2384                         readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
2385                 }
2386
2387         dev_dbg(css->dev, "%s: imgu 0x%x main 0x%x sp0 0x%x sp1 0x%x\n",
2388                 __func__, imgu_status, irq_status[IMGU_IRQCTRL_MAIN],
2389                 irq_status[IMGU_IRQCTRL_SP0], irq_status[IMGU_IRQCTRL_SP1]);
2390
2391         if (!imgu_status && !irq_status[IMGU_IRQCTRL_MAIN])
2392                 return -ENOMSG;
2393
2394         return 0;
2395 }