drivers/gpu/drm/exynos/exynos_drm_g2d.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2012 Samsung Electronics Co.Ltd
   4  * Authors: Joonyoung Shim <jy0922.shim@samsung.com>
   5  */
   6
   7 #include <linux/clk.h>
   8 #include <linux/component.h>
   9 #include <linux/delay.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/err.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/io.h>
  14 #include <linux/kernel.h>
  15 #include <linux/of.h>
  16 #include <linux/platform_device.h>
  17 #include <linux/pm_runtime.h>
  18 #include <linux/slab.h>
  19 #include <linux/uaccess.h>
  20 #include <linux/workqueue.h>
  21
  22 #include <drm/drm_file.h>
  23 #include <drm/exynos_drm.h>
  24
  25 #include "exynos_drm_drv.h"
  26 #include "exynos_drm_g2d.h"
  27 #include "exynos_drm_gem.h"
  28
  29 #define G2D_HW_MAJOR_VER                4
  30 #define G2D_HW_MINOR_VER                1
  31
  32 /* vaild register range set from user: 0x0104 ~ 0x0880 */
  33 #define G2D_VALID_START                 0x0104
  34 #define G2D_VALID_END                   0x0880
  35
  36 /* general registers */
  37 #define G2D_SOFT_RESET                  0x0000
  38 #define G2D_INTEN                       0x0004
  39 #define G2D_INTC_PEND                   0x000C
  40 #define G2D_DMA_SFR_BASE_ADDR           0x0080
  41 #define G2D_DMA_COMMAND                 0x0084
  42 #define G2D_DMA_STATUS                  0x008C
  43 #define G2D_DMA_HOLD_CMD                0x0090
  44
  45 /* command registers */
  46 #define G2D_BITBLT_START                0x0100
  47
  48 /* registers for base address */
  49 #define G2D_SRC_BASE_ADDR               0x0304
  50 #define G2D_SRC_STRIDE                  0x0308
  51 #define G2D_SRC_COLOR_MODE              0x030C
  52 #define G2D_SRC_LEFT_TOP                0x0310
  53 #define G2D_SRC_RIGHT_BOTTOM            0x0314
  54 #define G2D_SRC_PLANE2_BASE_ADDR        0x0318
  55 #define G2D_DST_BASE_ADDR               0x0404
  56 #define G2D_DST_STRIDE                  0x0408
  57 #define G2D_DST_COLOR_MODE              0x040C
  58 #define G2D_DST_LEFT_TOP                0x0410
  59 #define G2D_DST_RIGHT_BOTTOM            0x0414
  60 #define G2D_DST_PLANE2_BASE_ADDR        0x0418
  61 #define G2D_PAT_BASE_ADDR               0x0500
  62 #define G2D_MSK_BASE_ADDR               0x0520
  63
  64 /* G2D_SOFT_RESET */
  65 #define G2D_SFRCLEAR                    (1 << 1)
  66 #define G2D_R                           (1 << 0)
  67
  68 /* G2D_INTEN */
  69 #define G2D_INTEN_ACF                   (1 << 3)
  70 #define G2D_INTEN_UCF                   (1 << 2)
  71 #define G2D_INTEN_GCF                   (1 << 1)
  72 #define G2D_INTEN_SCF                   (1 << 0)
  73
  74 /* G2D_INTC_PEND */
  75 #define G2D_INTP_ACMD_FIN               (1 << 3)
  76 #define G2D_INTP_UCMD_FIN               (1 << 2)
  77 #define G2D_INTP_GCMD_FIN               (1 << 1)
  78 #define G2D_INTP_SCMD_FIN               (1 << 0)
  79
  80 /* G2D_DMA_COMMAND */
  81 #define G2D_DMA_HALT                    (1 << 2)
  82 #define G2D_DMA_CONTINUE                (1 << 1)
  83 #define G2D_DMA_START                   (1 << 0)
  84
  85 /* G2D_DMA_STATUS */
  86 #define G2D_DMA_LIST_DONE_COUNT         (0xFF << 17)
  87 #define G2D_DMA_BITBLT_DONE_COUNT       (0xFFFF << 1)
  88 #define G2D_DMA_DONE                    (1 << 0)
  89 #define G2D_DMA_LIST_DONE_COUNT_OFFSET  17
  90
  91 /* G2D_DMA_HOLD_CMD */
  92 #define G2D_USER_HOLD                   (1 << 2)
  93 #define G2D_LIST_HOLD                   (1 << 1)
  94 #define G2D_BITBLT_HOLD                 (1 << 0)
  95
  96 /* G2D_BITBLT_START */
  97 #define G2D_START_CASESEL               (1 << 2)
  98 #define G2D_START_NHOLT                 (1 << 1)
  99 #define G2D_START_BITBLT                (1 << 0)
 100
 101 /* buffer color format */
 102 #define G2D_FMT_XRGB8888                0
 103 #define G2D_FMT_ARGB8888                1
 104 #define G2D_FMT_RGB565                  2
 105 #define G2D_FMT_XRGB1555                3
 106 #define G2D_FMT_ARGB1555                4
 107 #define G2D_FMT_XRGB4444                5
 108 #define G2D_FMT_ARGB4444                6
 109 #define G2D_FMT_PACKED_RGB888           7
 110 #define G2D_FMT_A8                      11
 111 #define G2D_FMT_L8                      12
 112
 113 /* buffer valid length */
 114 #define G2D_LEN_MIN                     1
 115 #define G2D_LEN_MAX                     8000
 116
 117 #define G2D_CMDLIST_SIZE                (PAGE_SIZE / 4)
 118 #define G2D_CMDLIST_NUM                 64
 119 #define G2D_CMDLIST_POOL_SIZE           (G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM)
 120 #define G2D_CMDLIST_DATA_NUM            (G2D_CMDLIST_SIZE / sizeof(u32) - 2)
 121
 122 /* maximum buffer pool size of userptr is 64MB as default */
 123 #define MAX_POOL                (64 * 1024 * 1024)
 124
 125 enum {
 126         BUF_TYPE_GEM = 1,
 127         BUF_TYPE_USERPTR,
 128 };
 129
 130 enum g2d_reg_type {
 131         REG_TYPE_NONE = -1,
 132         REG_TYPE_SRC,
 133         REG_TYPE_SRC_PLANE2,
 134         REG_TYPE_DST,
 135         REG_TYPE_DST_PLANE2,
 136         REG_TYPE_PAT,
 137         REG_TYPE_MSK,
 138         MAX_REG_TYPE_NR
 139 };
 140
 141 enum g2d_flag_bits {
 142         /*
 143          * If set, suspends the runqueue worker after the currently
 144          * processed node is finished.
 145          */
 146         G2D_BIT_SUSPEND_RUNQUEUE,
 147         /*
 148          * If set, indicates that the engine is currently busy.
 149          */
 150         G2D_BIT_ENGINE_BUSY,
 151 };
 152
 153 /* cmdlist data structure */
 154 struct g2d_cmdlist {
 155         u32             head;
 156         unsigned long   data[G2D_CMDLIST_DATA_NUM];
 157         u32             last;   /* last data offset */
 158 };
 159
 160 /*
 161  * A structure of buffer description
 162  *
 163  * @format: color format
 164  * @stride: buffer stride/pitch in bytes
 165  * @left_x: the x coordinates of left top corner
 166  * @top_y: the y coordinates of left top corner
 167  * @right_x: the x coordinates of right bottom corner
 168  * @bottom_y: the y coordinates of right bottom corner
 169  *
 170  */
 171 struct g2d_buf_desc {
 172         unsigned int    format;
 173         unsigned int    stride;
 174         unsigned int    left_x;
 175         unsigned int    top_y;
 176         unsigned int    right_x;
 177         unsigned int    bottom_y;
 178 };
 179
 180 /*
 181  * A structure of buffer information
 182  *
 183  * @map_nr: manages the number of mapped buffers
 184  * @reg_types: stores regitster type in the order of requested command
 185  * @handles: stores buffer handle in its reg_type position
 186  * @types: stores buffer type in its reg_type position
 187  * @descs: stores buffer description in its reg_type position
 188  *
 189  */
 190 struct g2d_buf_info {
 191         unsigned int            map_nr;
 192         enum g2d_reg_type       reg_types[MAX_REG_TYPE_NR];
 193         void                    *obj[MAX_REG_TYPE_NR];
 194         unsigned int            types[MAX_REG_TYPE_NR];
 195         struct g2d_buf_desc     descs[MAX_REG_TYPE_NR];
 196 };
 197
 198 struct drm_exynos_pending_g2d_event {
 199         struct drm_pending_event        base;
 200         struct drm_exynos_g2d_event     event;
 201 };
 202
 203 struct g2d_cmdlist_userptr {
 204         struct list_head        list;
 205         dma_addr_t              dma_addr;
 206         unsigned long           userptr;
 207         unsigned long           size;
 208         struct frame_vector     *vec;
 209         struct sg_table         *sgt;
 210         atomic_t                refcount;
 211         bool                    in_pool;
 212         bool                    out_of_list;
 213 };
 214 struct g2d_cmdlist_node {
 215         struct list_head        list;
 216         struct g2d_cmdlist      *cmdlist;
 217         dma_addr_t              dma_addr;
 218         struct g2d_buf_info     buf_info;
 219
 220         struct drm_exynos_pending_g2d_event     *event;
 221 };
 222
 223 struct g2d_runqueue_node {
 224         struct list_head        list;
 225         struct list_head        run_cmdlist;
 226         struct list_head        event_list;
 227         struct drm_file         *filp;
 228         pid_t                   pid;
 229         struct completion       complete;
 230         int                     async;
 231 };
 232
 233 struct g2d_data {
 234         struct device                   *dev;
 235         struct clk                      *gate_clk;
 236         void __iomem                    *regs;
 237         int                             irq;
 238         struct workqueue_struct         *g2d_workq;
 239         struct work_struct              runqueue_work;
 240         struct drm_device               *drm_dev;
 241         unsigned long                   flags;
 242
 243         /* cmdlist */
 244         struct g2d_cmdlist_node         *cmdlist_node;
 245         struct list_head                free_cmdlist;
 246         struct mutex                    cmdlist_mutex;
 247         dma_addr_t                      cmdlist_pool;
 248         void                            *cmdlist_pool_virt;
 249         unsigned long                   cmdlist_dma_attrs;
 250
 251         /* runqueue*/
 252         struct g2d_runqueue_node        *runqueue_node;
 253         struct list_head                runqueue;
 254         struct mutex                    runqueue_mutex;
 255         struct kmem_cache               *runqueue_slab;
 256
 257         unsigned long                   current_pool;
 258         unsigned long                   max_pool;
 259 };
 260
 261 static inline void g2d_hw_reset(struct g2d_data *g2d)
 262 {
 263         writel(G2D_R | G2D_SFRCLEAR, g2d->regs + G2D_SOFT_RESET);
 264         clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
 265 }
 266
 267 static int g2d_init_cmdlist(struct g2d_data *g2d)
 268 {
 269         struct device *dev = g2d->dev;
 270         struct g2d_cmdlist_node *node;
 271         int nr;
 272         int ret;
 273         struct g2d_buf_info *buf_info;
 274
 275         g2d->cmdlist_dma_attrs = DMA_ATTR_WRITE_COMBINE;
 276
 277         g2d->cmdlist_pool_virt = dma_alloc_attrs(to_dma_dev(g2d->drm_dev),
 278                                                 G2D_CMDLIST_POOL_SIZE,
 279                                                 &g2d->cmdlist_pool, GFP_KERNEL,
 280                                                 g2d->cmdlist_dma_attrs);
 281         if (!g2d->cmdlist_pool_virt) {
 282                 dev_err(dev, "failed to allocate dma memory\n");
 283                 return -ENOMEM;
 284         }
 285
 286         node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL);
 287         if (!node) {
 288                 ret = -ENOMEM;
 289                 goto err;
 290         }
 291
 292         for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) {
 293                 unsigned int i;
 294
 295                 node[nr].cmdlist =
 296                         g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE;
 297                 node[nr].dma_addr =
 298                         g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE;
 299
 300                 buf_info = &node[nr].buf_info;
 301                 for (i = 0; i < MAX_REG_TYPE_NR; i++)
 302                         buf_info->reg_types[i] = REG_TYPE_NONE;
 303
 304                 list_add_tail(&node[nr].list, &g2d->free_cmdlist);
 305         }
 306
 307         return 0;
 308
 309 err:
 310         dma_free_attrs(to_dma_dev(g2d->drm_dev), G2D_CMDLIST_POOL_SIZE,
 311                         g2d->cmdlist_pool_virt,
 312                         g2d->cmdlist_pool, g2d->cmdlist_dma_attrs);
 313         return ret;
 314 }
 315
 316 static void g2d_fini_cmdlist(struct g2d_data *g2d)
 317 {
 318         kfree(g2d->cmdlist_node);
 319
 320         if (g2d->cmdlist_pool_virt && g2d->cmdlist_pool) {
 321                 dma_free_attrs(to_dma_dev(g2d->drm_dev),
 322                                 G2D_CMDLIST_POOL_SIZE,
 323                                 g2d->cmdlist_pool_virt,
 324                                 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs);
 325         }
 326 }
 327
 328 static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d)
 329 {
 330         struct device *dev = g2d->dev;
 331         struct g2d_cmdlist_node *node;
 332
 333         mutex_lock(&g2d->cmdlist_mutex);
 334         if (list_empty(&g2d->free_cmdlist)) {
 335                 dev_err(dev, "there is no free cmdlist\n");
 336                 mutex_unlock(&g2d->cmdlist_mutex);
 337                 return NULL;
 338         }
 339
 340         node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node,
 341                                 list);
 342         list_del_init(&node->list);
 343         mutex_unlock(&g2d->cmdlist_mutex);
 344
 345         return node;
 346 }
 347
 348 static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node)
 349 {
 350         mutex_lock(&g2d->cmdlist_mutex);
 351         list_move_tail(&node->list, &g2d->free_cmdlist);
 352         mutex_unlock(&g2d->cmdlist_mutex);
 353 }
 354
 355 static void g2d_add_cmdlist_to_inuse(struct drm_exynos_file_private *file_priv,
 356                                      struct g2d_cmdlist_node *node)
 357 {
 358         struct g2d_cmdlist_node *lnode;
 359
 360         if (list_empty(&file_priv->inuse_cmdlist))
 361                 goto add_to_list;
 362
 363         /* this links to base address of new cmdlist */
 364         lnode = list_entry(file_priv->inuse_cmdlist.prev,
 365                                 struct g2d_cmdlist_node, list);
 366         lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr;
 367
 368 add_to_list:
 369         list_add_tail(&node->list, &file_priv->inuse_cmdlist);
 370
 371         if (node->event)
 372                 list_add_tail(&node->event->base.link, &file_priv->event_list);
 373 }
 374
 375 static void g2d_userptr_put_dma_addr(struct g2d_data *g2d,
 376                                         void *obj,
 377                                         bool force)
 378 {
 379         struct g2d_cmdlist_userptr *g2d_userptr = obj;
 380         struct page **pages;
 381
 382         if (!obj)
 383                 return;
 384
 385         if (force)
 386                 goto out;
 387
 388         atomic_dec(&g2d_userptr->refcount);
 389
 390         if (atomic_read(&g2d_userptr->refcount) > 0)
 391                 return;
 392
 393         if (g2d_userptr->in_pool)
 394                 return;
 395
 396 out:
 397         dma_unmap_sg(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt->sgl,
 398                         g2d_userptr->sgt->nents, DMA_BIDIRECTIONAL);
 399
 400         pages = frame_vector_pages(g2d_userptr->vec);
 401         if (!IS_ERR(pages)) {
 402                 int i;
 403
 404                 for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++)
 405                         set_page_dirty_lock(pages[i]);
 406         }
 407         put_vaddr_frames(g2d_userptr->vec);
 408         frame_vector_destroy(g2d_userptr->vec);
 409
 410         if (!g2d_userptr->out_of_list)
 411                 list_del_init(&g2d_userptr->list);
 412
 413         sg_free_table(g2d_userptr->sgt);
 414         kfree(g2d_userptr->sgt);
 415         kfree(g2d_userptr);
 416 }
 417
 418 static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d,
 419                                         unsigned long userptr,
 420                                         unsigned long size,
 421                                         struct drm_file *filp,
 422                                         void **obj)
 423 {
 424         struct drm_exynos_file_private *file_priv = filp->driver_priv;
 425         struct g2d_cmdlist_userptr *g2d_userptr;
 426         struct sg_table *sgt;
 427         unsigned long start, end;
 428         unsigned int npages, offset;
 429         int ret;
 430
 431         if (!size) {
 432                 DRM_DEV_ERROR(g2d->dev, "invalid userptr size.\n");
 433                 return ERR_PTR(-EINVAL);
 434         }
 435
 436         /* check if userptr already exists in userptr_list. */
 437         list_for_each_entry(g2d_userptr, &file_priv->userptr_list, list) {
 438                 if (g2d_userptr->userptr == userptr) {
 439                         /*
 440                          * also check size because there could be same address
 441                          * and different size.
 442                          */
 443                         if (g2d_userptr->size == size) {
 444                                 atomic_inc(&g2d_userptr->refcount);
 445                                 *obj = g2d_userptr;
 446
 447                                 return &g2d_userptr->dma_addr;
 448                         }
 449
 450                         /*
 451                          * at this moment, maybe g2d dma is accessing this
 452                          * g2d_userptr memory region so just remove this
 453                          * g2d_userptr object from userptr_list not to be
 454                          * referred again and also except it the userptr
 455                          * pool to be released after the dma access completion.
 456                          */
 457                         g2d_userptr->out_of_list = true;
 458                         g2d_userptr->in_pool = false;
 459                         list_del_init(&g2d_userptr->list);
 460
 461                         break;
 462                 }
 463         }
 464
 465         g2d_userptr = kzalloc(sizeof(*g2d_userptr), GFP_KERNEL);
 466         if (!g2d_userptr)
 467                 return ERR_PTR(-ENOMEM);
 468
 469         atomic_set(&g2d_userptr->refcount, 1);
 470         g2d_userptr->size = size;
 471
 472         start = userptr & PAGE_MASK;
 473         offset = userptr & ~PAGE_MASK;
 474         end = PAGE_ALIGN(userptr + size);
 475         npages = (end - start) >> PAGE_SHIFT;
 476         g2d_userptr->vec = frame_vector_create(npages);
 477         if (!g2d_userptr->vec) {
 478                 ret = -ENOMEM;
 479                 goto err_free;
 480         }
 481
 482         ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE,
 483                 g2d_userptr->vec);
 484         if (ret != npages) {
 485                 DRM_DEV_ERROR(g2d->dev,
 486                               "failed to get user pages from userptr.\n");
 487                 if (ret < 0)
 488                         goto err_destroy_framevec;
 489                 ret = -EFAULT;
 490                 goto err_put_framevec;
 491         }
 492         if (frame_vector_to_pages(g2d_userptr->vec) < 0) {
 493                 ret = -EFAULT;
 494                 goto err_put_framevec;
 495         }
 496
 497         sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
 498         if (!sgt) {
 499                 ret = -ENOMEM;
 500                 goto err_put_framevec;
 501         }
 502
 503         ret = sg_alloc_table_from_pages(sgt,
 504                                         frame_vector_pages(g2d_userptr->vec),
 505                                         npages, offset, size, GFP_KERNEL);
 506         if (ret < 0) {
 507                 DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n");
 508                 goto err_free_sgt;
 509         }
 510
 511         g2d_userptr->sgt = sgt;
 512
 513         if (!dma_map_sg(to_dma_dev(g2d->drm_dev), sgt->sgl, sgt->nents,
 514                                 DMA_BIDIRECTIONAL)) {
 515                 DRM_DEV_ERROR(g2d->dev, "failed to map sgt with dma region.\n");
 516                 ret = -ENOMEM;
 517                 goto err_sg_free_table;
 518         }
 519
 520         g2d_userptr->dma_addr = sgt->sgl[0].dma_address;
 521         g2d_userptr->userptr = userptr;
 522
 523         list_add_tail(&g2d_userptr->list, &file_priv->userptr_list);
 524
 525         if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) {
 526                 g2d->current_pool += npages << PAGE_SHIFT;
 527                 g2d_userptr->in_pool = true;
 528         }
 529
 530         *obj = g2d_userptr;
 531
 532         return &g2d_userptr->dma_addr;
 533
 534 err_sg_free_table:
 535         sg_free_table(sgt);
 536
 537 err_free_sgt:
 538         kfree(sgt);
 539
 540 err_put_framevec:
 541         put_vaddr_frames(g2d_userptr->vec);
 542
 543 err_destroy_framevec:
 544         frame_vector_destroy(g2d_userptr->vec);
 545
 546 err_free:
 547         kfree(g2d_userptr);
 548
 549         return ERR_PTR(ret);
 550 }
 551
 552 static void g2d_userptr_free_all(struct g2d_data *g2d, struct drm_file *filp)
 553 {
 554         struct drm_exynos_file_private *file_priv = filp->driver_priv;
 555         struct g2d_cmdlist_userptr *g2d_userptr, *n;
 556
 557         list_for_each_entry_safe(g2d_userptr, n, &file_priv->userptr_list, list)
 558                 if (g2d_userptr->in_pool)
 559                         g2d_userptr_put_dma_addr(g2d, g2d_userptr, true);
 560
 561         g2d->current_pool = 0;
 562 }
 563
 564 static enum g2d_reg_type g2d_get_reg_type(struct g2d_data *g2d, int reg_offset)
 565 {
 566         enum g2d_reg_type reg_type;
 567
 568         switch (reg_offset) {
 569         case G2D_SRC_BASE_ADDR:
 570         case G2D_SRC_STRIDE:
 571         case G2D_SRC_COLOR_MODE:
 572         case G2D_SRC_LEFT_TOP:
 573         case G2D_SRC_RIGHT_BOTTOM:
 574                 reg_type = REG_TYPE_SRC;
 575                 break;
 576         case G2D_SRC_PLANE2_BASE_ADDR:
 577                 reg_type = REG_TYPE_SRC_PLANE2;
 578                 break;
 579         case G2D_DST_BASE_ADDR:
 580         case G2D_DST_STRIDE:
 581         case G2D_DST_COLOR_MODE:
 582         case G2D_DST_LEFT_TOP:
 583         case G2D_DST_RIGHT_BOTTOM:
 584                 reg_type = REG_TYPE_DST;
 585                 break;
 586         case G2D_DST_PLANE2_BASE_ADDR:
 587                 reg_type = REG_TYPE_DST_PLANE2;
 588                 break;
 589         case G2D_PAT_BASE_ADDR:
 590                 reg_type = REG_TYPE_PAT;
 591                 break;
 592         case G2D_MSK_BASE_ADDR:
 593                 reg_type = REG_TYPE_MSK;
 594                 break;
 595         default:
 596                 reg_type = REG_TYPE_NONE;
 597                 DRM_DEV_ERROR(g2d->dev, "Unknown register offset![%d]\n",
 598                               reg_offset);
 599                 break;
 600         }
 601
 602         return reg_type;
 603 }
 604
 605 static unsigned long g2d_get_buf_bpp(unsigned int format)
 606 {
 607         unsigned long bpp;
 608
 609         switch (format) {
 610         case G2D_FMT_XRGB8888:
 611         case G2D_FMT_ARGB8888:
 612                 bpp = 4;
 613                 break;
 614         case G2D_FMT_RGB565:
 615         case G2D_FMT_XRGB1555:
 616         case G2D_FMT_ARGB1555:
 617         case G2D_FMT_XRGB4444:
 618         case G2D_FMT_ARGB4444:
 619                 bpp = 2;
 620                 break;
 621         case G2D_FMT_PACKED_RGB888:
 622                 bpp = 3;
 623                 break;
 624         default:
 625                 bpp = 1;
 626                 break;
 627         }
 628
 629         return bpp;
 630 }
 631
 632 static bool g2d_check_buf_desc_is_valid(struct g2d_data *g2d,
 633                                         struct g2d_buf_desc *buf_desc,
 634                                         enum g2d_reg_type reg_type,
 635                                         unsigned long size)
 636 {
 637         int width, height;
 638         unsigned long bpp, last_pos;
 639
 640         /*
 641          * check source and destination buffers only.
 642          * so the others are always valid.
 643          */
 644         if (reg_type != REG_TYPE_SRC && reg_type != REG_TYPE_DST)
 645                 return true;
 646
 647         /* This check also makes sure that right_x > left_x. */
 648         width = (int)buf_desc->right_x - (int)buf_desc->left_x;
 649         if (width < G2D_LEN_MIN || width > G2D_LEN_MAX) {
 650                 DRM_DEV_ERROR(g2d->dev, "width[%d] is out of range!\n", width);
 651                 return false;
 652         }
 653
 654         /* This check also makes sure that bottom_y > top_y. */
 655         height = (int)buf_desc->bottom_y - (int)buf_desc->top_y;
 656         if (height < G2D_LEN_MIN || height > G2D_LEN_MAX) {
 657                 DRM_DEV_ERROR(g2d->dev,
 658                               "height[%d] is out of range!\n", height);
 659                 return false;
 660         }
 661
 662         bpp = g2d_get_buf_bpp(buf_desc->format);
 663
 664         /* Compute the position of the last byte that the engine accesses. */
 665         last_pos = ((unsigned long)buf_desc->bottom_y - 1) *
 666                 (unsigned long)buf_desc->stride +
 667                 (unsigned long)buf_desc->right_x * bpp - 1;
 668
 669         /*
 670          * Since right_x > left_x and bottom_y > top_y we already know
 671          * that the first_pos < last_pos (first_pos being the position
 672          * of the first byte the engine accesses), it just remains to
 673          * check if last_pos is smaller then the buffer size.
 674          */
 675
 676         if (last_pos >= size) {
 677                 DRM_DEV_ERROR(g2d->dev, "last engine access position [%lu] "
 678                               "is out of range [%lu]!\n", last_pos, size);
 679                 return false;
 680         }
 681
 682         return true;
 683 }
 684
 685 static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
 686                                 struct g2d_cmdlist_node *node,
 687                                 struct drm_device *drm_dev,
 688                                 struct drm_file *file)
 689 {
 690         struct g2d_cmdlist *cmdlist = node->cmdlist;
 691         struct g2d_buf_info *buf_info = &node->buf_info;
 692         int offset;
 693         int ret;
 694         int i;
 695
 696         for (i = 0; i < buf_info->map_nr; i++) {
 697                 struct g2d_buf_desc *buf_desc;
 698                 enum g2d_reg_type reg_type;
 699                 int reg_pos;
 700                 unsigned long handle;
 701                 dma_addr_t *addr;
 702
 703                 reg_pos = cmdlist->last - 2 * (i + 1);
 704
 705                 offset = cmdlist->data[reg_pos];
 706                 handle = cmdlist->data[reg_pos + 1];
 707
 708                 reg_type = g2d_get_reg_type(g2d, offset);
 709                 if (reg_type == REG_TYPE_NONE) {
 710                         ret = -EFAULT;
 711                         goto err;
 712                 }
 713
 714                 buf_desc = &buf_info->descs[reg_type];
 715
 716                 if (buf_info->types[reg_type] == BUF_TYPE_GEM) {
 717                         struct exynos_drm_gem *exynos_gem;
 718
 719                         exynos_gem = exynos_drm_gem_get(file, handle);
 720                         if (!exynos_gem) {
 721                                 ret = -EFAULT;
 722                                 goto err;
 723                         }
 724
 725                         if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,
 726                                                          reg_type, exynos_gem->size)) {
 727                                 exynos_drm_gem_put(exynos_gem);
 728                                 ret = -EFAULT;
 729                                 goto err;
 730                         }
 731
 732                         addr = &exynos_gem->dma_addr;
 733                         buf_info->obj[reg_type] = exynos_gem;
 734                 } else {
 735                         struct drm_exynos_g2d_userptr g2d_userptr;
 736
 737                         if (copy_from_user(&g2d_userptr, (void __user *)handle,
 738                                 sizeof(struct drm_exynos_g2d_userptr))) {
 739                                 ret = -EFAULT;
 740                                 goto err;
 741                         }
 742
 743                         if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,
 744                                                          reg_type,
 745                                                          g2d_userptr.size)) {
 746                                 ret = -EFAULT;
 747                                 goto err;
 748                         }
 749
 750                         addr = g2d_userptr_get_dma_addr(g2d,
 751                                                         g2d_userptr.userptr,
 752                                                         g2d_userptr.size,
 753                                                         file,
 754                                                         &buf_info->obj[reg_type]);
 755                         if (IS_ERR(addr)) {
 756                                 ret = -EFAULT;
 757                                 goto err;
 758                         }
 759                 }
 760
 761                 cmdlist->data[reg_pos + 1] = *addr;
 762                 buf_info->reg_types[i] = reg_type;
 763         }
 764
 765         return 0;
 766
 767 err:
 768         buf_info->map_nr = i;
 769         return ret;
 770 }
 771
 772 static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d,
 773                                   struct g2d_cmdlist_node *node,
 774                                   struct drm_file *filp)
 775 {
 776         struct g2d_buf_info *buf_info = &node->buf_info;
 777         int i;
 778
 779         for (i = 0; i < buf_info->map_nr; i++) {
 780                 struct g2d_buf_desc *buf_desc;
 781                 enum g2d_reg_type reg_type;
 782                 void *obj;
 783
 784                 reg_type = buf_info->reg_types[i];
 785
 786                 buf_desc = &buf_info->descs[reg_type];
 787                 obj = buf_info->obj[reg_type];
 788
 789                 if (buf_info->types[reg_type] == BUF_TYPE_GEM)
 790                         exynos_drm_gem_put(obj);
 791                 else
 792                         g2d_userptr_put_dma_addr(g2d, obj, false);
 793
 794                 buf_info->reg_types[i] = REG_TYPE_NONE;
 795                 buf_info->obj[reg_type] = NULL;
 796                 buf_info->types[reg_type] = 0;
 797                 memset(buf_desc, 0x00, sizeof(*buf_desc));
 798         }
 799
 800         buf_info->map_nr = 0;
 801 }
 802
 803 static void g2d_dma_start(struct g2d_data *g2d,
 804                           struct g2d_runqueue_node *runqueue_node)
 805 {
 806         struct g2d_cmdlist_node *node =
 807                                 list_first_entry(&runqueue_node->run_cmdlist,
 808                                                 struct g2d_cmdlist_node, list);
 809
 810         set_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
 811         writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR);
 812         writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND);
 813 }
 814
 815 static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d)
 816 {
 817         struct g2d_runqueue_node *runqueue_node;
 818
 819         if (list_empty(&g2d->runqueue))
 820                 return NULL;
 821
 822         runqueue_node = list_first_entry(&g2d->runqueue,
 823                                          struct g2d_runqueue_node, list);
 824         list_del_init(&runqueue_node->list);
 825         return runqueue_node;
 826 }
 827
 828 static void g2d_free_runqueue_node(struct g2d_data *g2d,
 829                                    struct g2d_runqueue_node *runqueue_node)
 830 {
 831         struct g2d_cmdlist_node *node;
 832
 833         mutex_lock(&g2d->cmdlist_mutex);
 834         /*
 835          * commands in run_cmdlist have been completed so unmap all gem
 836          * objects in each command node so that they are unreferenced.
 837          */
 838         list_for_each_entry(node, &runqueue_node->run_cmdlist, list)
 839                 g2d_unmap_cmdlist_gem(g2d, node, runqueue_node->filp);
 840         list_splice_tail_init(&runqueue_node->run_cmdlist, &g2d->free_cmdlist);
 841         mutex_unlock(&g2d->cmdlist_mutex);
 842
 843         kmem_cache_free(g2d->runqueue_slab, runqueue_node);
 844 }
 845
 846 /**
 847  * g2d_remove_runqueue_nodes - remove items from the list of runqueue nodes
 848  * @g2d: G2D state object
 849  * @file: if not zero, only remove items with this DRM file
 850  *
 851  * Has to be called under runqueue lock.
 852  */
 853 static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file *file)
 854 {
 855         struct g2d_runqueue_node *node, *n;
 856
 857         if (list_empty(&g2d->runqueue))
 858                 return;
 859
 860         list_for_each_entry_safe(node, n, &g2d->runqueue, list) {
 861                 if (file && node->filp != file)
 862                         continue;
 863
 864                 list_del_init(&node->list);
 865                 g2d_free_runqueue_node(g2d, node);
 866         }
 867 }
 868
 869 static void g2d_runqueue_worker(struct work_struct *work)
 870 {
 871         struct g2d_data *g2d = container_of(work, struct g2d_data,
 872                                             runqueue_work);
 873         struct g2d_runqueue_node *runqueue_node;
 874
 875         /*
 876          * The engine is busy and the completion of the current node is going
 877          * to poke the runqueue worker, so nothing to do here.
 878          */
 879         if (test_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags))
 880                 return;
 881
 882         mutex_lock(&g2d->runqueue_mutex);
 883
 884         runqueue_node = g2d->runqueue_node;
 885         g2d->runqueue_node = NULL;
 886
 887         if (runqueue_node) {
 888                 pm_runtime_mark_last_busy(g2d->dev);
 889                 pm_runtime_put_autosuspend(g2d->dev);
 890
 891                 complete(&runqueue_node->complete);
 892                 if (runqueue_node->async)
 893                         g2d_free_runqueue_node(g2d, runqueue_node);
 894         }
 895
 896         if (!test_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags)) {
 897                 g2d->runqueue_node = g2d_get_runqueue_node(g2d);
 898
 899                 if (g2d->runqueue_node) {
 900                         pm_runtime_get_sync(g2d->dev);
 901                         g2d_dma_start(g2d, g2d->runqueue_node);
 902                 }
 903         }
 904
 905         mutex_unlock(&g2d->runqueue_mutex);
 906 }
 907
 908 static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no)
 909 {
 910         struct drm_device *drm_dev = g2d->drm_dev;
 911         struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node;
 912         struct drm_exynos_pending_g2d_event *e;
 913         struct timespec64 now;
 914
 915         if (list_empty(&runqueue_node->event_list))
 916                 return;
 917
 918         e = list_first_entry(&runqueue_node->event_list,
 919                              struct drm_exynos_pending_g2d_event, base.link);
 920
 921         ktime_get_ts64(&now);
 922         e->event.tv_sec = now.tv_sec;
 923         e->event.tv_usec = now.tv_nsec / NSEC_PER_USEC;
 924         e->event.cmdlist_no = cmdlist_no;
 925
 926         drm_send_event(drm_dev, &e->base);
 927 }
 928
 929 static irqreturn_t g2d_irq_handler(int irq, void *dev_id)
 930 {
 931         struct g2d_data *g2d = dev_id;
 932         u32 pending;
 933
 934         pending = readl_relaxed(g2d->regs + G2D_INTC_PEND);
 935         if (pending)
 936                 writel_relaxed(pending, g2d->regs + G2D_INTC_PEND);
 937
 938         if (pending & G2D_INTP_GCMD_FIN) {
 939                 u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS);
 940
 941                 cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >>
 942                                                 G2D_DMA_LIST_DONE_COUNT_OFFSET;
 943
 944                 g2d_finish_event(g2d, cmdlist_no);
 945
 946                 writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD);
 947                 if (!(pending & G2D_INTP_ACMD_FIN)) {
 948                         writel_relaxed(G2D_DMA_CONTINUE,
 949                                         g2d->regs + G2D_DMA_COMMAND);
 950                 }
 951         }
 952
 953         if (pending & G2D_INTP_ACMD_FIN) {
 954                 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
 955                 queue_work(g2d->g2d_workq, &g2d->runqueue_work);
 956         }
 957
 958         return IRQ_HANDLED;
 959 }
 960
 961 /**
 962  * g2d_wait_finish - wait for the G2D engine to finish the current runqueue node
 963  * @g2d: G2D state object
 964  * @file: if not zero, only wait if the current runqueue node belongs
 965  *        to the DRM file
 966  *
 967  * Should the engine not become idle after a 100ms timeout, a hardware
 968  * reset is issued.
 969  */
 970 static void g2d_wait_finish(struct g2d_data *g2d, struct drm_file *file)
 971 {
 972         struct device *dev = g2d->dev;
 973
 974         struct g2d_runqueue_node *runqueue_node = NULL;
 975         unsigned int tries = 10;
 976
 977         mutex_lock(&g2d->runqueue_mutex);
 978
 979         /* If no node is currently processed, we have nothing to do. */
 980         if (!g2d->runqueue_node)
 981                 goto out;
 982
 983         runqueue_node = g2d->runqueue_node;
 984
 985         /* Check if the currently processed item belongs to us. */
 986         if (file && runqueue_node->filp != file)
 987                 goto out;
 988
 989         mutex_unlock(&g2d->runqueue_mutex);
 990
 991         /* Wait for the G2D engine to finish. */
 992         while (tries-- && (g2d->runqueue_node == runqueue_node))
 993                 mdelay(10);
 994
 995         mutex_lock(&g2d->runqueue_mutex);
 996
 997         if (g2d->runqueue_node != runqueue_node)
 998                 goto out;
 999
1000         dev_err(dev, "wait timed out, resetting engine...\n");
1001         g2d_hw_reset(g2d);
1002
1003         /*
1004          * After the hardware reset of the engine we are going to loose
1005          * the IRQ which triggers the PM runtime put().
1006          * So do this manually here.
1007          */
1008         pm_runtime_mark_last_busy(dev);
1009         pm_runtime_put_autosuspend(dev);
1010
1011         complete(&runqueue_node->complete);
1012         if (runqueue_node->async)
1013                 g2d_free_runqueue_node(g2d, runqueue_node);
1014
1015 out:
1016         mutex_unlock(&g2d->runqueue_mutex);
1017 }
1018
1019 static int g2d_check_reg_offset(struct g2d_data *g2d,
1020                                 struct g2d_cmdlist_node *node,
1021                                 int nr, bool for_addr)
1022 {
1023         struct g2d_cmdlist *cmdlist = node->cmdlist;
1024         int reg_offset;
1025         int index;
1026         int i;
1027
1028         for (i = 0; i < nr; i++) {
1029                 struct g2d_buf_info *buf_info = &node->buf_info;
1030                 struct g2d_buf_desc *buf_desc;
1031                 enum g2d_reg_type reg_type;
1032                 unsigned long value;
1033
1034                 index = cmdlist->last - 2 * (i + 1);
1035
1036                 reg_offset = cmdlist->data[index] & ~0xfffff000;
1037                 if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END)
1038                         goto err;
1039                 if (reg_offset % 4)
1040                         goto err;
1041
1042                 switch (reg_offset) {
1043                 case G2D_SRC_BASE_ADDR:
1044                 case G2D_SRC_PLANE2_BASE_ADDR:
1045                 case G2D_DST_BASE_ADDR:
1046                 case G2D_DST_PLANE2_BASE_ADDR:
1047                 case G2D_PAT_BASE_ADDR:
1048                 case G2D_MSK_BASE_ADDR:
1049                         if (!for_addr)
1050                                 goto err;
1051
1052                         reg_type = g2d_get_reg_type(g2d, reg_offset);
1053
1054                         /* check userptr buffer type. */
1055                         if ((cmdlist->data[index] & ~0x7fffffff) >> 31) {
1056                                 buf_info->types[reg_type] = BUF_TYPE_USERPTR;
1057                                 cmdlist->data[index] &= ~G2D_BUF_USERPTR;
1058                         } else
1059                                 buf_info->types[reg_type] = BUF_TYPE_GEM;
1060                         break;
1061                 case G2D_SRC_STRIDE:
1062                 case G2D_DST_STRIDE:
1063                         if (for_addr)
1064                                 goto err;
1065
1066                         reg_type = g2d_get_reg_type(g2d, reg_offset);
1067
1068                         buf_desc = &buf_info->descs[reg_type];
1069                         buf_desc->stride = cmdlist->data[index + 1];
1070                         break;
1071                 case G2D_SRC_COLOR_MODE:
1072                 case G2D_DST_COLOR_MODE:
1073                         if (for_addr)
1074                                 goto err;
1075
1076                         reg_type = g2d_get_reg_type(g2d, reg_offset);
1077
1078                         buf_desc = &buf_info->descs[reg_type];
1079                         value = cmdlist->data[index + 1];
1080
1081                         buf_desc->format = value & 0xf;
1082                         break;
1083                 case G2D_SRC_LEFT_TOP:
1084                 case G2D_DST_LEFT_TOP:
1085                         if (for_addr)
1086                                 goto err;
1087
1088                         reg_type = g2d_get_reg_type(g2d, reg_offset);
1089
1090                         buf_desc = &buf_info->descs[reg_type];
1091                         value = cmdlist->data[index + 1];
1092
1093                         buf_desc->left_x = value & 0x1fff;
1094                         buf_desc->top_y = (value & 0x1fff0000) >> 16;
1095                         break;
1096                 case G2D_SRC_RIGHT_BOTTOM:
1097                 case G2D_DST_RIGHT_BOTTOM:
1098                         if (for_addr)
1099                                 goto err;
1100
1101                         reg_type = g2d_get_reg_type(g2d, reg_offset);
1102
1103                         buf_desc = &buf_info->descs[reg_type];
1104                         value = cmdlist->data[index + 1];
1105
1106                         buf_desc->right_x = value & 0x1fff;
1107                         buf_desc->bottom_y = (value & 0x1fff0000) >> 16;
1108                         break;
1109                 default:
1110                         if (for_addr)
1111                                 goto err;
1112                         break;
1113                 }
1114         }
1115
1116         return 0;
1117
1118 err:
1119         dev_err(g2d->dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]);
1120         return -EINVAL;
1121 }
1122
1123 /* ioctl functions */
1124 int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data,
1125                              struct drm_file *file)
1126 {
1127         struct drm_exynos_g2d_get_ver *ver = data;
1128
1129         ver->major = G2D_HW_MAJOR_VER;
1130         ver->minor = G2D_HW_MINOR_VER;
1131
1132         return 0;
1133 }
1134
1135 int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
1136                                  struct drm_file *file)
1137 {
1138         struct drm_exynos_file_private *file_priv = file->driver_priv;
1139         struct exynos_drm_private *priv = drm_dev->dev_private;
1140         struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev);
1141         struct drm_exynos_g2d_set_cmdlist *req = data;
1142         struct drm_exynos_g2d_cmd *cmd;
1143         struct drm_exynos_pending_g2d_event *e;
1144         struct g2d_cmdlist_node *node;
1145         struct g2d_cmdlist *cmdlist;
1146         int size;
1147         int ret;
1148
1149         node = g2d_get_cmdlist(g2d);
1150         if (!node)
1151                 return -ENOMEM;
1152
1153         /*
1154          * To avoid an integer overflow for the later size computations, we
1155          * enforce a maximum number of submitted commands here. This limit is
1156          * sufficient for all conceivable usage cases of the G2D.
1157          */
1158         if (req->cmd_nr > G2D_CMDLIST_DATA_NUM ||
1159             req->cmd_buf_nr > G2D_CMDLIST_DATA_NUM) {
1160                 dev_err(g2d->dev, "number of submitted G2D commands exceeds limit\n");
1161                 return -EINVAL;
1162         }
1163
1164         node->event = NULL;
1165
1166         if (req->event_type != G2D_EVENT_NOT) {
1167                 e = kzalloc(sizeof(*node->event), GFP_KERNEL);
1168                 if (!e) {
1169                         ret = -ENOMEM;
1170                         goto err;
1171                 }
1172
1173                 e->event.base.type = DRM_EXYNOS_G2D_EVENT;
1174                 e->event.base.length = sizeof(e->event);
1175                 e->event.user_data = req->user_data;
1176
1177                 ret = drm_event_reserve_init(drm_dev, file, &e->base, &e->event.base);
1178                 if (ret) {
1179                         kfree(e);
1180                         goto err;
1181                 }
1182
1183                 node->event = e;
1184         }
1185
1186         cmdlist = node->cmdlist;
1187
1188         cmdlist->last = 0;
1189
1190         /*
1191          * If don't clear SFR registers, the cmdlist is affected by register
1192          * values of previous cmdlist. G2D hw executes SFR clear command and
1193          * a next command at the same time then the next command is ignored and
1194          * is executed rightly from next next command, so needs a dummy command
1195          * to next command of SFR clear command.
1196          */
1197         cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET;
1198         cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR;
1199         cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR;
1200         cmdlist->data[cmdlist->last++] = 0;
1201
1202         /*
1203          * 'LIST_HOLD' command should be set to the DMA_HOLD_CMD_REG
1204          * and GCF bit should be set to INTEN register if user wants
1205          * G2D interrupt event once current command list execution is
1206          * finished.
1207          * Otherwise only ACF bit should be set to INTEN register so
1208          * that one interrupt is occurred after all command lists
1209          * have been completed.
1210          */
1211         if (node->event) {
1212                 cmdlist->data[cmdlist->last++] = G2D_INTEN;
1213                 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF | G2D_INTEN_GCF;
1214                 cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD;
1215                 cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD;
1216         } else {
1217                 cmdlist->data[cmdlist->last++] = G2D_INTEN;
1218                 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF;
1219         }
1220
1221         /*
1222          * Check the size of cmdlist. The 2 that is added last comes from
1223          * the implicit G2D_BITBLT_START that is appended once we have
1224          * checked all the submitted commands.
1225          */
1226         size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2;
1227         if (size > G2D_CMDLIST_DATA_NUM) {
1228                 dev_err(g2d->dev, "cmdlist size is too big\n");
1229                 ret = -EINVAL;
1230                 goto err_free_event;
1231         }
1232
1233         cmd = (struct drm_exynos_g2d_cmd *)(unsigned long)req->cmd;
1234
1235         if (copy_from_user(cmdlist->data + cmdlist->last,
1236                                 (void __user *)cmd,
1237                                 sizeof(*cmd) * req->cmd_nr)) {
1238                 ret = -EFAULT;
1239                 goto err_free_event;
1240         }
1241         cmdlist->last += req->cmd_nr * 2;
1242
1243         ret = g2d_check_reg_offset(g2d, node, req->cmd_nr, false);
1244         if (ret < 0)
1245                 goto err_free_event;
1246
1247         node->buf_info.map_nr = req->cmd_buf_nr;
1248         if (req->cmd_buf_nr) {
1249                 struct drm_exynos_g2d_cmd *cmd_buf;
1250
1251                 cmd_buf = (struct drm_exynos_g2d_cmd *)
1252                                 (unsigned long)req->cmd_buf;
1253
1254                 if (copy_from_user(cmdlist->data + cmdlist->last,
1255                                         (void __user *)cmd_buf,
1256                                         sizeof(*cmd_buf) * req->cmd_buf_nr)) {
1257                         ret = -EFAULT;
1258                         goto err_free_event;
1259                 }
1260                 cmdlist->last += req->cmd_buf_nr * 2;
1261
1262                 ret = g2d_check_reg_offset(g2d, node, req->cmd_buf_nr, true);
1263                 if (ret < 0)
1264                         goto err_free_event;
1265
1266                 ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file);
1267                 if (ret < 0)
1268                         goto err_unmap;
1269         }
1270
1271         cmdlist->data[cmdlist->last++] = G2D_BITBLT_START;
1272         cmdlist->data[cmdlist->last++] = G2D_START_BITBLT;
1273
1274         /* head */
1275         cmdlist->head = cmdlist->last / 2;
1276
1277         /* tail */
1278         cmdlist->data[cmdlist->last] = 0;
1279
1280         g2d_add_cmdlist_to_inuse(file_priv, node);
1281
1282         return 0;
1283
1284 err_unmap:
1285         g2d_unmap_cmdlist_gem(g2d, node, file);
1286 err_free_event:
1287         if (node->event)
1288                 drm_event_cancel_free(drm_dev, &node->event->base);
1289 err:
1290         g2d_put_cmdlist(g2d, node);
1291         return ret;
1292 }
1293
1294 int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,
1295                           struct drm_file *file)
1296 {
1297         struct drm_exynos_file_private *file_priv = file->driver_priv;
1298         struct exynos_drm_private *priv = drm_dev->dev_private;
1299         struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev);
1300         struct drm_exynos_g2d_exec *req = data;
1301         struct g2d_runqueue_node *runqueue_node;
1302         struct list_head *run_cmdlist;
1303         struct list_head *event_list;
1304
1305         runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL);
1306         if (!runqueue_node)
1307                 return -ENOMEM;
1308
1309         run_cmdlist = &runqueue_node->run_cmdlist;
1310         event_list = &runqueue_node->event_list;
1311         INIT_LIST_HEAD(run_cmdlist);
1312         INIT_LIST_HEAD(event_list);
1313         init_completion(&runqueue_node->complete);
1314         runqueue_node->async = req->async;
1315
1316         list_splice_init(&file_priv->inuse_cmdlist, run_cmdlist);
1317         list_splice_init(&file_priv->event_list, event_list);
1318
1319         if (list_empty(run_cmdlist)) {
1320                 dev_err(g2d->dev, "there is no inuse cmdlist\n");
1321                 kmem_cache_free(g2d->runqueue_slab, runqueue_node);
1322                 return -EPERM;
1323         }
1324
1325         mutex_lock(&g2d->runqueue_mutex);
1326         runqueue_node->pid = current->pid;
1327         runqueue_node->filp = file;
1328         list_add_tail(&runqueue_node->list, &g2d->runqueue);
1329         mutex_unlock(&g2d->runqueue_mutex);
1330
1331         /* Let the runqueue know that there is work to do. */
1332         queue_work(g2d->g2d_workq, &g2d->runqueue_work);
1333
1334         if (runqueue_node->async)
1335                 goto out;
1336
1337         wait_for_completion(&runqueue_node->complete);
1338         g2d_free_runqueue_node(g2d, runqueue_node);
1339
1340 out:
1341         return 0;
1342 }
1343
1344 int g2d_open(struct drm_device *drm_dev, struct drm_file *file)
1345 {
1346         struct drm_exynos_file_private *file_priv = file->driver_priv;
1347
1348         INIT_LIST_HEAD(&file_priv->inuse_cmdlist);
1349         INIT_LIST_HEAD(&file_priv->event_list);
1350         INIT_LIST_HEAD(&file_priv->userptr_list);
1351
1352         return 0;
1353 }
1354
1355 void g2d_close(struct drm_device *drm_dev, struct drm_file *file)
1356 {
1357         struct drm_exynos_file_private *file_priv = file->driver_priv;
1358         struct exynos_drm_private *priv = drm_dev->dev_private;
1359         struct g2d_data *g2d;
1360         struct g2d_cmdlist_node *node, *n;
1361
1362         if (!priv->g2d_dev)
1363                 return;
1364
1365         g2d = dev_get_drvdata(priv->g2d_dev);
1366
1367         /* Remove the runqueue nodes that belong to us. */
1368         mutex_lock(&g2d->runqueue_mutex);
1369         g2d_remove_runqueue_nodes(g2d, file);
1370         mutex_unlock(&g2d->runqueue_mutex);
1371
1372         /*
1373          * Wait for the runqueue worker to finish its current node.
1374          * After this the engine should no longer be accessing any
1375          * memory belonging to us.
1376          */
1377         g2d_wait_finish(g2d, file);
1378
1379         /*
1380          * Even after the engine is idle, there might still be stale cmdlists
1381          * (i.e. cmdlisst which we submitted but never executed) around, with
1382          * their corresponding GEM/userptr buffers.
1383          * Properly unmap these buffers here.
1384          */
1385         mutex_lock(&g2d->cmdlist_mutex);
1386         list_for_each_entry_safe(node, n, &file_priv->inuse_cmdlist, list) {
1387                 g2d_unmap_cmdlist_gem(g2d, node, file);
1388                 list_move_tail(&node->list, &g2d->free_cmdlist);
1389         }
1390         mutex_unlock(&g2d->cmdlist_mutex);
1391
1392         /* release all g2d_userptr in pool. */
1393         g2d_userptr_free_all(g2d, file);
1394 }
1395
1396 static int g2d_bind(struct device *dev, struct device *master, void *data)
1397 {
1398         struct g2d_data *g2d = dev_get_drvdata(dev);
1399         struct drm_device *drm_dev = data;
1400         struct exynos_drm_private *priv = drm_dev->dev_private;
1401         int ret;
1402
1403         g2d->drm_dev = drm_dev;
1404
1405         /* allocate dma-aware cmdlist buffer. */
1406         ret = g2d_init_cmdlist(g2d);
1407         if (ret < 0) {
1408                 dev_err(dev, "cmdlist init failed\n");
1409                 return ret;
1410         }
1411
1412         ret = exynos_drm_register_dma(drm_dev, dev);
1413         if (ret < 0) {
1414                 dev_err(dev, "failed to enable iommu.\n");
1415                 g2d_fini_cmdlist(g2d);
1416                 return ret;
1417         }
1418         priv->g2d_dev = dev;
1419
1420         dev_info(dev, "The Exynos G2D (ver %d.%d) successfully registered.\n",
1421                         G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER);
1422         return 0;
1423 }
1424
1425 static void g2d_unbind(struct device *dev, struct device *master, void *data)
1426 {
1427         struct g2d_data *g2d = dev_get_drvdata(dev);
1428         struct drm_device *drm_dev = data;
1429         struct exynos_drm_private *priv = drm_dev->dev_private;
1430
1431         /* Suspend operation and wait for engine idle. */
1432         set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1433         g2d_wait_finish(g2d, NULL);
1434         priv->g2d_dev = NULL;
1435
1436         cancel_work_sync(&g2d->runqueue_work);
1437         exynos_drm_unregister_dma(g2d->drm_dev, dev);
1438 }
1439
1440 static const struct component_ops g2d_component_ops = {
1441         .bind   = g2d_bind,
1442         .unbind = g2d_unbind,
1443 };
1444
1445 static int g2d_probe(struct platform_device *pdev)
1446 {
1447         struct device *dev = &pdev->dev;
1448         struct resource *res;
1449         struct g2d_data *g2d;
1450         int ret;
1451
1452         g2d = devm_kzalloc(dev, sizeof(*g2d), GFP_KERNEL);
1453         if (!g2d)
1454                 return -ENOMEM;
1455
1456         g2d->runqueue_slab = kmem_cache_create("g2d_runqueue_slab",
1457                         sizeof(struct g2d_runqueue_node), 0, 0, NULL);
1458         if (!g2d->runqueue_slab)
1459                 return -ENOMEM;
1460
1461         g2d->dev = dev;
1462
1463         g2d->g2d_workq = create_singlethread_workqueue("g2d");
1464         if (!g2d->g2d_workq) {
1465                 dev_err(dev, "failed to create workqueue\n");
1466                 ret = -EINVAL;
1467                 goto err_destroy_slab;
1468         }
1469
1470         INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker);
1471         INIT_LIST_HEAD(&g2d->free_cmdlist);
1472         INIT_LIST_HEAD(&g2d->runqueue);
1473
1474         mutex_init(&g2d->cmdlist_mutex);
1475         mutex_init(&g2d->runqueue_mutex);
1476
1477         g2d->gate_clk = devm_clk_get(dev, "fimg2d");
1478         if (IS_ERR(g2d->gate_clk)) {
1479                 dev_err(dev, "failed to get gate clock\n");
1480                 ret = PTR_ERR(g2d->gate_clk);
1481                 goto err_destroy_workqueue;
1482         }
1483
1484         pm_runtime_use_autosuspend(dev);
1485         pm_runtime_set_autosuspend_delay(dev, 2000);
1486         pm_runtime_enable(dev);
1487         clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1488         clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
1489
1490         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1491
1492         g2d->regs = devm_ioremap_resource(dev, res);
1493         if (IS_ERR(g2d->regs)) {
1494                 ret = PTR_ERR(g2d->regs);
1495                 goto err_put_clk;
1496         }
1497
1498         g2d->irq = platform_get_irq(pdev, 0);
1499         if (g2d->irq < 0) {
1500                 dev_err(dev, "failed to get irq\n");
1501                 ret = g2d->irq;
1502                 goto err_put_clk;
1503         }
1504
1505         ret = devm_request_irq(dev, g2d->irq, g2d_irq_handler, 0,
1506                                                                 "drm_g2d", g2d);
1507         if (ret < 0) {
1508                 dev_err(dev, "irq request failed\n");
1509                 goto err_put_clk;
1510         }
1511
1512         g2d->max_pool = MAX_POOL;
1513
1514         platform_set_drvdata(pdev, g2d);
1515
1516         ret = component_add(dev, &g2d_component_ops);
1517         if (ret < 0) {
1518                 dev_err(dev, "failed to register drm g2d device\n");
1519                 goto err_put_clk;
1520         }
1521
1522         return 0;
1523
1524 err_put_clk:
1525         pm_runtime_disable(dev);
1526 err_destroy_workqueue:
1527         destroy_workqueue(g2d->g2d_workq);
1528 err_destroy_slab:
1529         kmem_cache_destroy(g2d->runqueue_slab);
1530         return ret;
1531 }
1532
1533 static int g2d_remove(struct platform_device *pdev)
1534 {
1535         struct g2d_data *g2d = platform_get_drvdata(pdev);
1536
1537         component_del(&pdev->dev, &g2d_component_ops);
1538
1539         /* There should be no locking needed here. */
1540         g2d_remove_runqueue_nodes(g2d, NULL);
1541
1542         pm_runtime_dont_use_autosuspend(&pdev->dev);
1543         pm_runtime_disable(&pdev->dev);
1544
1545         g2d_fini_cmdlist(g2d);
1546         destroy_workqueue(g2d->g2d_workq);
1547         kmem_cache_destroy(g2d->runqueue_slab);
1548
1549         return 0;
1550 }
1551
1552 #ifdef CONFIG_PM_SLEEP
1553 static int g2d_suspend(struct device *dev)
1554 {
1555         struct g2d_data *g2d = dev_get_drvdata(dev);
1556
1557         /*
1558          * Suspend the runqueue worker operation and wait until the G2D
1559          * engine is idle.
1560          */
1561         set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1562         g2d_wait_finish(g2d, NULL);
1563         flush_work(&g2d->runqueue_work);
1564
1565         return 0;
1566 }
1567
1568 static int g2d_resume(struct device *dev)
1569 {
1570         struct g2d_data *g2d = dev_get_drvdata(dev);
1571
1572         clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1573         queue_work(g2d->g2d_workq, &g2d->runqueue_work);
1574
1575         return 0;
1576 }
1577 #endif
1578
1579 #ifdef CONFIG_PM
1580 static int g2d_runtime_suspend(struct device *dev)
1581 {
1582         struct g2d_data *g2d = dev_get_drvdata(dev);
1583
1584         clk_disable_unprepare(g2d->gate_clk);
1585
1586         return 0;
1587 }
1588
1589 static int g2d_runtime_resume(struct device *dev)
1590 {
1591         struct g2d_data *g2d = dev_get_drvdata(dev);
1592         int ret;
1593
1594         ret = clk_prepare_enable(g2d->gate_clk);
1595         if (ret < 0)
1596                 dev_warn(dev, "failed to enable clock.\n");
1597
1598         return ret;
1599 }
1600 #endif
1601
1602 static const struct dev_pm_ops g2d_pm_ops = {
1603         SET_SYSTEM_SLEEP_PM_OPS(g2d_suspend, g2d_resume)
1604         SET_RUNTIME_PM_OPS(g2d_runtime_suspend, g2d_runtime_resume, NULL)
1605 };
1606
1607 static const struct of_device_id exynos_g2d_match[] = {
1608         { .compatible = "samsung,exynos5250-g2d" },
1609         { .compatible = "samsung,exynos4212-g2d" },
1610         {},
1611 };
1612 MODULE_DEVICE_TABLE(of, exynos_g2d_match);
1613
1614 struct platform_driver g2d_driver = {
1615         .probe          = g2d_probe,
1616         .remove         = g2d_remove,
1617         .driver         = {
1618                 .name   = "exynos-drm-g2d",
1619                 .owner  = THIS_MODULE,
1620                 .pm     = &g2d_pm_ops,
1621                 .of_match_table = exynos_g2d_match,
1622         },
1623 };