drivers/gpu/drm/i915/i915_sw_fence.c

   1 /*
   2  * (C) Copyright 2016 Intel Corporation
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License
   6  * as published by the Free Software Foundation; version 2
   7  * of the License.
   8  */
   9
  10 #include <linux/slab.h>
  11 #include <linux/dma-fence.h>
  12 #include <linux/irq_work.h>
  13 #include <linux/reservation.h>
  14
  15 #include "i915_sw_fence.h"
  16 #include "i915_selftest.h"
  17
  18 #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */
  19
  20 static DEFINE_SPINLOCK(i915_sw_fence_lock);
  21
  22 enum {
  23         DEBUG_FENCE_IDLE = 0,
  24         DEBUG_FENCE_NOTIFY,
  25 };
  26
  27 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
  28
  29 static void *i915_sw_fence_debug_hint(void *addr)
  30 {
  31         return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK);
  32 }
  33
  34 static struct debug_obj_descr i915_sw_fence_debug_descr = {
  35         .name = "i915_sw_fence",
  36         .debug_hint = i915_sw_fence_debug_hint,
  37 };
  38
  39 static inline void debug_fence_init(struct i915_sw_fence *fence)
  40 {
  41         debug_object_init(fence, &i915_sw_fence_debug_descr);
  42 }
  43
  44 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
  45 {
  46         debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);
  47 }
  48
  49 static inline void debug_fence_activate(struct i915_sw_fence *fence)
  50 {
  51         debug_object_activate(fence, &i915_sw_fence_debug_descr);
  52 }
  53
  54 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
  55                                          int old, int new)
  56 {
  57         debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
  58 }
  59
  60 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
  61 {
  62         debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
  63 }
  64
  65 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
  66 {
  67         debug_object_destroy(fence, &i915_sw_fence_debug_descr);
  68 }
  69
  70 static inline void debug_fence_free(struct i915_sw_fence *fence)
  71 {
  72         debug_object_free(fence, &i915_sw_fence_debug_descr);
  73         smp_wmb(); /* flush the change in state before reallocation */
  74 }
  75
  76 static inline void debug_fence_assert(struct i915_sw_fence *fence)
  77 {
  78         debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
  79 }
  80
  81 #else
  82
  83 static inline void debug_fence_init(struct i915_sw_fence *fence)
  84 {
  85 }
  86
  87 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
  88 {
  89 }
  90
  91 static inline void debug_fence_activate(struct i915_sw_fence *fence)
  92 {
  93 }
  94
  95 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
  96                                          int old, int new)
  97 {
  98 }
  99
 100 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
 101 {
 102 }
 103
 104 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
 105 {
 106 }
 107
 108 static inline void debug_fence_free(struct i915_sw_fence *fence)
 109 {
 110 }
 111
 112 static inline void debug_fence_assert(struct i915_sw_fence *fence)
 113 {
 114 }
 115
 116 #endif
 117
 118 static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
 119                                   enum i915_sw_fence_notify state)
 120 {
 121         i915_sw_fence_notify_t fn;
 122
 123         fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
 124         return fn(fence, state);
 125 }
 126
 127 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
 128 void i915_sw_fence_fini(struct i915_sw_fence *fence)
 129 {
 130         debug_fence_free(fence);
 131 }
 132 #endif
 133
 134 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
 135                                         struct list_head *continuation)
 136 {
 137         wait_queue_head_t *x = &fence->wait;
 138         wait_queue_entry_t *pos, *next;
 139         unsigned long flags;
 140
 141         debug_fence_deactivate(fence);
 142         atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
 143
 144         /*
 145          * To prevent unbounded recursion as we traverse the graph of
 146          * i915_sw_fences, we move the entry list from this, the next ready
 147          * fence, to the tail of the original fence's entry list
 148          * (and so added to the list to be woken).
 149          */
 150
 151         spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
 152         if (continuation) {
 153                 list_for_each_entry_safe(pos, next, &x->head, entry) {
 154                         if (pos->func == autoremove_wake_function)
 155                                 pos->func(pos, TASK_NORMAL, 0, continuation);
 156                         else
 157                                 list_move_tail(&pos->entry, continuation);
 158                 }
 159         } else {
 160                 LIST_HEAD(extra);
 161
 162                 do {
 163                         list_for_each_entry_safe(pos, next, &x->head, entry)
 164                                 pos->func(pos, TASK_NORMAL, 0, &extra);
 165
 166                         if (list_empty(&extra))
 167                                 break;
 168
 169                         list_splice_tail_init(&extra, &x->head);
 170                 } while (1);
 171         }
 172         spin_unlock_irqrestore(&x->lock, flags);
 173
 174         debug_fence_assert(fence);
 175 }
 176
 177 static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
 178                                      struct list_head *continuation)
 179 {
 180         debug_fence_assert(fence);
 181
 182         if (!atomic_dec_and_test(&fence->pending))
 183                 return;
 184
 185         debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
 186
 187         if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
 188                 return;
 189
 190         debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
 191
 192         __i915_sw_fence_wake_up_all(fence, continuation);
 193
 194         debug_fence_destroy(fence);
 195         __i915_sw_fence_notify(fence, FENCE_FREE);
 196 }
 197
 198 static void i915_sw_fence_complete(struct i915_sw_fence *fence)
 199 {
 200         debug_fence_assert(fence);
 201
 202         if (WARN_ON(i915_sw_fence_done(fence)))
 203                 return;
 204
 205         __i915_sw_fence_complete(fence, NULL);
 206 }
 207
 208 static void i915_sw_fence_await(struct i915_sw_fence *fence)
 209 {
 210         debug_fence_assert(fence);
 211         WARN_ON(atomic_inc_return(&fence->pending) <= 1);
 212 }
 213
 214 void __i915_sw_fence_init(struct i915_sw_fence *fence,
 215                           i915_sw_fence_notify_t fn,
 216                           const char *name,
 217                           struct lock_class_key *key)
 218 {
 219         BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK);
 220
 221         debug_fence_init(fence);
 222
 223         __init_waitqueue_head(&fence->wait, name, key);
 224         atomic_set(&fence->pending, 1);
 225         fence->flags = (unsigned long)fn;
 226 }
 227
 228 void i915_sw_fence_commit(struct i915_sw_fence *fence)
 229 {
 230         debug_fence_activate(fence);
 231         i915_sw_fence_complete(fence);
 232 }
 233
 234 static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
 235 {
 236         list_del(&wq->entry);
 237         __i915_sw_fence_complete(wq->private, key);
 238
 239         if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
 240                 kfree(wq);
 241         return 0;
 242 }
 243
 244 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 245                                     const struct i915_sw_fence * const signaler)
 246 {
 247         wait_queue_entry_t *wq;
 248
 249         if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
 250                 return false;
 251
 252         if (fence == signaler)
 253                 return true;
 254
 255         list_for_each_entry(wq, &fence->wait.head, entry) {
 256                 if (wq->func != i915_sw_fence_wake)
 257                         continue;
 258
 259                 if (__i915_sw_fence_check_if_after(wq->private, signaler))
 260                         return true;
 261         }
 262
 263         return false;
 264 }
 265
 266 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
 267 {
 268         wait_queue_entry_t *wq;
 269
 270         if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
 271                 return;
 272
 273         list_for_each_entry(wq, &fence->wait.head, entry) {
 274                 if (wq->func != i915_sw_fence_wake)
 275                         continue;
 276
 277                 __i915_sw_fence_clear_checked_bit(wq->private);
 278         }
 279 }
 280
 281 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 282                                   const struct i915_sw_fence * const signaler)
 283 {
 284         unsigned long flags;
 285         bool err;
 286
 287         if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
 288                 return false;
 289
 290         spin_lock_irqsave(&i915_sw_fence_lock, flags);
 291         err = __i915_sw_fence_check_if_after(fence, signaler);
 292         __i915_sw_fence_clear_checked_bit(fence);
 293         spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
 294
 295         return err;
 296 }
 297
 298 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 299                                           struct i915_sw_fence *signaler,
 300                                           wait_queue_entry_t *wq, gfp_t gfp)
 301 {
 302         unsigned long flags;
 303         int pending;
 304
 305         debug_fence_assert(fence);
 306         might_sleep_if(gfpflags_allow_blocking(gfp));
 307
 308         if (i915_sw_fence_done(signaler))
 309                 return 0;
 310
 311         debug_fence_assert(signaler);
 312
 313         /* The dependency graph must be acyclic. */
 314         if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
 315                 return -EINVAL;
 316
 317         pending = 0;
 318         if (!wq) {
 319                 wq = kmalloc(sizeof(*wq), gfp);
 320                 if (!wq) {
 321                         if (!gfpflags_allow_blocking(gfp))
 322                                 return -ENOMEM;
 323
 324                         i915_sw_fence_wait(signaler);
 325                         return 0;
 326                 }
 327
 328                 pending |= I915_SW_FENCE_FLAG_ALLOC;
 329         }
 330
 331         INIT_LIST_HEAD(&wq->entry);
 332         wq->flags = pending;
 333         wq->func = i915_sw_fence_wake;
 334         wq->private = fence;
 335
 336         i915_sw_fence_await(fence);
 337
 338         spin_lock_irqsave(&signaler->wait.lock, flags);
 339         if (likely(!i915_sw_fence_done(signaler))) {
 340                 __add_wait_queue_entry_tail(&signaler->wait, wq);
 341                 pending = 1;
 342         } else {
 343                 i915_sw_fence_wake(wq, 0, 0, NULL);
 344                 pending = 0;
 345         }
 346         spin_unlock_irqrestore(&signaler->wait.lock, flags);
 347
 348         return pending;
 349 }
 350
 351 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 352                                  struct i915_sw_fence *signaler,
 353                                  wait_queue_entry_t *wq)
 354 {
 355         return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
 356 }
 357
 358 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
 359                                      struct i915_sw_fence *signaler,
 360                                      gfp_t gfp)
 361 {
 362         return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
 363 }
 364
 365 struct i915_sw_dma_fence_cb {
 366         struct dma_fence_cb base;
 367         struct i915_sw_fence *fence;
 368 };
 369
 370 struct i915_sw_dma_fence_cb_timer {
 371         struct i915_sw_dma_fence_cb base;
 372         struct dma_fence *dma;
 373         struct timer_list timer;
 374         struct irq_work work;
 375         struct rcu_head rcu;
 376 };
 377
 378 static void dma_i915_sw_fence_wake(struct dma_fence *dma,
 379                                    struct dma_fence_cb *data)
 380 {
 381         struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
 382
 383         i915_sw_fence_complete(cb->fence);
 384         kfree(cb);
 385 }
 386
 387 static void timer_i915_sw_fence_wake(struct timer_list *t)
 388 {
 389         struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer);
 390         struct i915_sw_fence *fence;
 391
 392         fence = xchg(&cb->base.fence, NULL);
 393         if (!fence)
 394                 return;
 395
 396         pr_warn("asynchronous wait on fence %s:%s:%x timed out\n",
 397                 cb->dma->ops->get_driver_name(cb->dma),
 398                 cb->dma->ops->get_timeline_name(cb->dma),
 399                 cb->dma->seqno);
 400
 401         i915_sw_fence_complete(fence);
 402 }
 403
 404 static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma,
 405                                          struct dma_fence_cb *data)
 406 {
 407         struct i915_sw_dma_fence_cb_timer *cb =
 408                 container_of(data, typeof(*cb), base.base);
 409         struct i915_sw_fence *fence;
 410
 411         fence = xchg(&cb->base.fence, NULL);
 412         if (fence)
 413                 i915_sw_fence_complete(fence);
 414
 415         irq_work_queue(&cb->work);
 416 }
 417
 418 static void irq_i915_sw_fence_work(struct irq_work *wrk)
 419 {
 420         struct i915_sw_dma_fence_cb_timer *cb =
 421                 container_of(wrk, typeof(*cb), work);
 422
 423         del_timer_sync(&cb->timer);
 424         dma_fence_put(cb->dma);
 425
 426         kfree_rcu(cb, rcu);
 427 }
 428
 429 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 430                                   struct dma_fence *dma,
 431                                   unsigned long timeout,
 432                                   gfp_t gfp)
 433 {
 434         struct i915_sw_dma_fence_cb *cb;
 435         dma_fence_func_t func;
 436         int ret;
 437
 438         debug_fence_assert(fence);
 439         might_sleep_if(gfpflags_allow_blocking(gfp));
 440
 441         if (dma_fence_is_signaled(dma))
 442                 return 0;
 443
 444         cb = kmalloc(timeout ?
 445                      sizeof(struct i915_sw_dma_fence_cb_timer) :
 446                      sizeof(struct i915_sw_dma_fence_cb),
 447                      gfp);
 448         if (!cb) {
 449                 if (!gfpflags_allow_blocking(gfp))
 450                         return -ENOMEM;
 451
 452                 return dma_fence_wait(dma, false);
 453         }
 454
 455         cb->fence = fence;
 456         i915_sw_fence_await(fence);
 457
 458         func = dma_i915_sw_fence_wake;
 459         if (timeout) {
 460                 struct i915_sw_dma_fence_cb_timer *timer =
 461                         container_of(cb, typeof(*timer), base);
 462
 463                 timer->dma = dma_fence_get(dma);
 464                 init_irq_work(&timer->work, irq_i915_sw_fence_work);
 465
 466                 timer_setup(&timer->timer,
 467                             timer_i915_sw_fence_wake, TIMER_IRQSAFE);
 468                 mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout));
 469
 470                 func = dma_i915_sw_fence_wake_timer;
 471         }
 472
 473         ret = dma_fence_add_callback(dma, &cb->base, func);
 474         if (ret == 0) {
 475                 ret = 1;
 476         } else {
 477                 func(dma, &cb->base);
 478                 if (ret == -ENOENT) /* fence already signaled */
 479                         ret = 0;
 480         }
 481
 482         return ret;
 483 }
 484
 485 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 486                                     struct reservation_object *resv,
 487                                     const struct dma_fence_ops *exclude,
 488                                     bool write,
 489                                     unsigned long timeout,
 490                                     gfp_t gfp)
 491 {
 492         struct dma_fence *excl;
 493         int ret = 0, pending;
 494
 495         debug_fence_assert(fence);
 496         might_sleep_if(gfpflags_allow_blocking(gfp));
 497
 498         if (write) {
 499                 struct dma_fence **shared;
 500                 unsigned int count, i;
 501
 502                 ret = reservation_object_get_fences_rcu(resv,
 503                                                         &excl, &count, &shared);
 504                 if (ret)
 505                         return ret;
 506
 507                 for (i = 0; i < count; i++) {
 508                         if (shared[i]->ops == exclude)
 509                                 continue;
 510
 511                         pending = i915_sw_fence_await_dma_fence(fence,
 512                                                                 shared[i],
 513                                                                 timeout,
 514                                                                 gfp);
 515                         if (pending < 0) {
 516                                 ret = pending;
 517                                 break;
 518                         }
 519
 520                         ret |= pending;
 521                 }
 522
 523                 for (i = 0; i < count; i++)
 524                         dma_fence_put(shared[i]);
 525                 kfree(shared);
 526         } else {
 527                 excl = reservation_object_get_excl_rcu(resv);
 528         }
 529
 530         if (ret >= 0 && excl && excl->ops != exclude) {
 531                 pending = i915_sw_fence_await_dma_fence(fence,
 532                                                         excl,
 533                                                         timeout,
 534                                                         gfp);
 535                 if (pending < 0)
 536                         ret = pending;
 537                 else
 538                         ret |= pending;
 539         }
 540
 541         dma_fence_put(excl);
 542
 543         return ret;
 544 }
 545
 546 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 547 #include "selftests/lib_sw_fence.c"
 548 #include "selftests/i915_sw_fence.c"
 549 #endif