drivers/gpu/drm/i915/gt/intel_engine_cs.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <drm/drm_print.h>
  26
  27 #include "gem/i915_gem_context.h"
  28
  29 #include "i915_drv.h"
  30
  31 #include "intel_engine.h"
  32 #include "intel_engine_pm.h"
  33 #include "intel_context.h"
  34 #include "intel_lrc.h"
  35 #include "intel_reset.h"
  36
  37 /* Haswell does have the CXT_SIZE register however it does not appear to be
  38  * valid. Now, docs explain in dwords what is in the context object. The full
  39  * size is 70720 bytes, however, the power context and execlist context will
  40  * never be saved (power context is stored elsewhere, and execlists don't work
  41  * on HSW) - so the final size, including the extra state required for the
  42  * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
  43  */
  44 #define HSW_CXT_TOTAL_SIZE              (17 * PAGE_SIZE)
  45
  46 #define DEFAULT_LR_CONTEXT_RENDER_SIZE  (22 * PAGE_SIZE)
  47 #define GEN8_LR_CONTEXT_RENDER_SIZE     (20 * PAGE_SIZE)
  48 #define GEN9_LR_CONTEXT_RENDER_SIZE     (22 * PAGE_SIZE)
  49 #define GEN10_LR_CONTEXT_RENDER_SIZE    (18 * PAGE_SIZE)
  50 #define GEN11_LR_CONTEXT_RENDER_SIZE    (14 * PAGE_SIZE)
  51
  52 #define GEN8_LR_CONTEXT_OTHER_SIZE      ( 2 * PAGE_SIZE)
  53
  54 struct engine_class_info {
  55         const char *name;
  56         u8 uabi_class;
  57 };
  58
  59 static const struct engine_class_info intel_engine_classes[] = {
  60         [RENDER_CLASS] = {
  61                 .name = "rcs",
  62                 .uabi_class = I915_ENGINE_CLASS_RENDER,
  63         },
  64         [COPY_ENGINE_CLASS] = {
  65                 .name = "bcs",
  66                 .uabi_class = I915_ENGINE_CLASS_COPY,
  67         },
  68         [VIDEO_DECODE_CLASS] = {
  69                 .name = "vcs",
  70                 .uabi_class = I915_ENGINE_CLASS_VIDEO,
  71         },
  72         [VIDEO_ENHANCEMENT_CLASS] = {
  73                 .name = "vecs",
  74                 .uabi_class = I915_ENGINE_CLASS_VIDEO_ENHANCE,
  75         },
  76 };
  77
  78 #define MAX_MMIO_BASES 3
  79 struct engine_info {
  80         unsigned int hw_id;
  81         u8 class;
  82         u8 instance;
  83         /* mmio bases table *must* be sorted in reverse gen order */
  84         struct engine_mmio_base {
  85                 u32 gen : 8;
  86                 u32 base : 24;
  87         } mmio_bases[MAX_MMIO_BASES];
  88 };
  89
  90 static const struct engine_info intel_engines[] = {
  91         [RCS0] = {
  92                 .hw_id = RCS0_HW,
  93                 .class = RENDER_CLASS,
  94                 .instance = 0,
  95                 .mmio_bases = {
  96                         { .gen = 1, .base = RENDER_RING_BASE }
  97                 },
  98         },
  99         [BCS0] = {
 100                 .hw_id = BCS0_HW,
 101                 .class = COPY_ENGINE_CLASS,
 102                 .instance = 0,
 103                 .mmio_bases = {
 104                         { .gen = 6, .base = BLT_RING_BASE }
 105                 },
 106         },
 107         [VCS0] = {
 108                 .hw_id = VCS0_HW,
 109                 .class = VIDEO_DECODE_CLASS,
 110                 .instance = 0,
 111                 .mmio_bases = {
 112                         { .gen = 11, .base = GEN11_BSD_RING_BASE },
 113                         { .gen = 6, .base = GEN6_BSD_RING_BASE },
 114                         { .gen = 4, .base = BSD_RING_BASE }
 115                 },
 116         },
 117         [VCS1] = {
 118                 .hw_id = VCS1_HW,
 119                 .class = VIDEO_DECODE_CLASS,
 120                 .instance = 1,
 121                 .mmio_bases = {
 122                         { .gen = 11, .base = GEN11_BSD2_RING_BASE },
 123                         { .gen = 8, .base = GEN8_BSD2_RING_BASE }
 124                 },
 125         },
 126         [VCS2] = {
 127                 .hw_id = VCS2_HW,
 128                 .class = VIDEO_DECODE_CLASS,
 129                 .instance = 2,
 130                 .mmio_bases = {
 131                         { .gen = 11, .base = GEN11_BSD3_RING_BASE }
 132                 },
 133         },
 134         [VCS3] = {
 135                 .hw_id = VCS3_HW,
 136                 .class = VIDEO_DECODE_CLASS,
 137                 .instance = 3,
 138                 .mmio_bases = {
 139                         { .gen = 11, .base = GEN11_BSD4_RING_BASE }
 140                 },
 141         },
 142         [VECS0] = {
 143                 .hw_id = VECS0_HW,
 144                 .class = VIDEO_ENHANCEMENT_CLASS,
 145                 .instance = 0,
 146                 .mmio_bases = {
 147                         { .gen = 11, .base = GEN11_VEBOX_RING_BASE },
 148                         { .gen = 7, .base = VEBOX_RING_BASE }
 149                 },
 150         },
 151         [VECS1] = {
 152                 .hw_id = VECS1_HW,
 153                 .class = VIDEO_ENHANCEMENT_CLASS,
 154                 .instance = 1,
 155                 .mmio_bases = {
 156                         { .gen = 11, .base = GEN11_VEBOX2_RING_BASE }
 157                 },
 158         },
 159 };
 160
 161 /**
 162  * intel_engine_context_size() - return the size of the context for an engine
 163  * @dev_priv: i915 device private
 164  * @class: engine class
 165  *
 166  * Each engine class may require a different amount of space for a context
 167  * image.
 168  *
 169  * Return: size (in bytes) of an engine class specific context image
 170  *
 171  * Note: this size includes the HWSP, which is part of the context image
 172  * in LRC mode, but does not include the "shared data page" used with
 173  * GuC submission. The caller should account for this if using the GuC.
 174  */
 175 u32 intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
 176 {
 177         u32 cxt_size;
 178
 179         BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE);
 180
 181         switch (class) {
 182         case RENDER_CLASS:
 183                 switch (INTEL_GEN(dev_priv)) {
 184                 default:
 185                         MISSING_CASE(INTEL_GEN(dev_priv));
 186                         return DEFAULT_LR_CONTEXT_RENDER_SIZE;
 187                 case 11:
 188                         return GEN11_LR_CONTEXT_RENDER_SIZE;
 189                 case 10:
 190                         return GEN10_LR_CONTEXT_RENDER_SIZE;
 191                 case 9:
 192                         return GEN9_LR_CONTEXT_RENDER_SIZE;
 193                 case 8:
 194                         return GEN8_LR_CONTEXT_RENDER_SIZE;
 195                 case 7:
 196                         if (IS_HASWELL(dev_priv))
 197                                 return HSW_CXT_TOTAL_SIZE;
 198
 199                         cxt_size = I915_READ(GEN7_CXT_SIZE);
 200                         return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,
 201                                         PAGE_SIZE);
 202                 case 6:
 203                         cxt_size = I915_READ(CXT_SIZE);
 204                         return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,
 205                                         PAGE_SIZE);
 206                 case 5:
 207                 case 4:
 208                         /*
 209                          * There is a discrepancy here between the size reported
 210                          * by the register and the size of the context layout
 211                          * in the docs. Both are described as authorative!
 212                          *
 213                          * The discrepancy is on the order of a few cachelines,
 214                          * but the total is under one page (4k), which is our
 215                          * minimum allocation anyway so it should all come
 216                          * out in the wash.
 217                          */
 218                         cxt_size = I915_READ(CXT_SIZE) + 1;
 219                         DRM_DEBUG_DRIVER("gen%d CXT_SIZE = %d bytes [0x%08x]\n",
 220                                          INTEL_GEN(dev_priv),
 221                                          cxt_size * 64,
 222                                          cxt_size - 1);
 223                         return round_up(cxt_size * 64, PAGE_SIZE);
 224                 case 3:
 225                 case 2:
 226                 /* For the special day when i810 gets merged. */
 227                 case 1:
 228                         return 0;
 229                 }
 230                 break;
 231         default:
 232                 MISSING_CASE(class);
 233                 /* fall through */
 234         case VIDEO_DECODE_CLASS:
 235         case VIDEO_ENHANCEMENT_CLASS:
 236         case COPY_ENGINE_CLASS:
 237                 if (INTEL_GEN(dev_priv) < 8)
 238                         return 0;
 239                 return GEN8_LR_CONTEXT_OTHER_SIZE;
 240         }
 241 }
 242
 243 static u32 __engine_mmio_base(struct drm_i915_private *i915,
 244                               const struct engine_mmio_base *bases)
 245 {
 246         int i;
 247
 248         for (i = 0; i < MAX_MMIO_BASES; i++)
 249                 if (INTEL_GEN(i915) >= bases[i].gen)
 250                         break;
 251
 252         GEM_BUG_ON(i == MAX_MMIO_BASES);
 253         GEM_BUG_ON(!bases[i].base);
 254
 255         return bases[i].base;
 256 }
 257
 258 static void __sprint_engine_name(char *name, const struct engine_info *info)
 259 {
 260         WARN_ON(snprintf(name, INTEL_ENGINE_CS_MAX_NAME, "%s%u",
 261                          intel_engine_classes[info->class].name,
 262                          info->instance) >= INTEL_ENGINE_CS_MAX_NAME);
 263 }
 264
 265 void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask)
 266 {
 267         /*
 268          * Though they added more rings on g4x/ilk, they did not add
 269          * per-engine HWSTAM until gen6.
 270          */
 271         if (INTEL_GEN(engine->i915) < 6 && engine->class != RENDER_CLASS)
 272                 return;
 273
 274         if (INTEL_GEN(engine->i915) >= 3)
 275                 ENGINE_WRITE(engine, RING_HWSTAM, mask);
 276         else
 277                 ENGINE_WRITE16(engine, RING_HWSTAM, mask);
 278 }
 279
 280 static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine)
 281 {
 282         /* Mask off all writes into the unknown HWSP */
 283         intel_engine_set_hwsp_writemask(engine, ~0u);
 284 }
 285
 286 static int
 287 intel_engine_setup(struct drm_i915_private *dev_priv,
 288                    enum intel_engine_id id)
 289 {
 290         const struct engine_info *info = &intel_engines[id];
 291         struct intel_engine_cs *engine;
 292
 293         GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes));
 294
 295         BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH));
 296         BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH));
 297
 298         if (GEM_DEBUG_WARN_ON(info->class > MAX_ENGINE_CLASS))
 299                 return -EINVAL;
 300
 301         if (GEM_DEBUG_WARN_ON(info->instance > MAX_ENGINE_INSTANCE))
 302                 return -EINVAL;
 303
 304         if (GEM_DEBUG_WARN_ON(dev_priv->engine_class[info->class][info->instance]))
 305                 return -EINVAL;
 306
 307         GEM_BUG_ON(dev_priv->engine[id]);
 308         engine = kzalloc(sizeof(*engine), GFP_KERNEL);
 309         if (!engine)
 310                 return -ENOMEM;
 311
 312         BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES);
 313
 314         engine->id = id;
 315         engine->mask = BIT(id);
 316         engine->i915 = dev_priv;
 317         engine->uncore = &dev_priv->uncore;
 318         __sprint_engine_name(engine->name, info);
 319         engine->hw_id = engine->guc_id = info->hw_id;
 320         engine->mmio_base = __engine_mmio_base(dev_priv, info->mmio_bases);
 321         engine->class = info->class;
 322         engine->instance = info->instance;
 323
 324         /*
 325          * To be overridden by the backend on setup. However to facilitate
 326          * cleanup on error during setup, we always provide the destroy vfunc.
 327          */
 328         engine->destroy = (typeof(engine->destroy))kfree;
 329
 330         engine->uabi_class = intel_engine_classes[info->class].uabi_class;
 331
 332         engine->context_size = intel_engine_context_size(dev_priv,
 333                                                          engine->class);
 334         if (WARN_ON(engine->context_size > BIT(20)))
 335                 engine->context_size = 0;
 336         if (engine->context_size)
 337                 DRIVER_CAPS(dev_priv)->has_logical_contexts = true;
 338
 339         /* Nothing to do here, execute in order of dependencies */
 340         engine->schedule = NULL;
 341
 342         seqlock_init(&engine->stats.lock);
 343
 344         ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
 345
 346         /* Scrub mmio state on takeover */
 347         intel_engine_sanitize_mmio(engine);
 348
 349         dev_priv->engine_class[info->class][info->instance] = engine;
 350         dev_priv->engine[id] = engine;
 351         return 0;
 352 }
 353
 354 static void __setup_engine_capabilities(struct intel_engine_cs *engine)
 355 {
 356         struct drm_i915_private *i915 = engine->i915;
 357
 358         if (engine->class == VIDEO_DECODE_CLASS) {
 359                 /*
 360                  * HEVC support is present on first engine instance
 361                  * before Gen11 and on all instances afterwards.
 362                  */
 363                 if (INTEL_GEN(i915) >= 11 ||
 364                     (INTEL_GEN(i915) >= 9 && engine->instance == 0))
 365                         engine->uabi_capabilities |=
 366                                 I915_VIDEO_CLASS_CAPABILITY_HEVC;
 367
 368                 /*
 369                  * SFC block is present only on even logical engine
 370                  * instances.
 371                  */
 372                 if ((INTEL_GEN(i915) >= 11 &&
 373                      RUNTIME_INFO(i915)->vdbox_sfc_access & engine->mask) ||
 374                     (INTEL_GEN(i915) >= 9 && engine->instance == 0))
 375                         engine->uabi_capabilities |=
 376                                 I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC;
 377         } else if (engine->class == VIDEO_ENHANCEMENT_CLASS) {
 378                 if (INTEL_GEN(i915) >= 9)
 379                         engine->uabi_capabilities |=
 380                                 I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC;
 381         }
 382 }
 383
 384 static void intel_setup_engine_capabilities(struct drm_i915_private *i915)
 385 {
 386         struct intel_engine_cs *engine;
 387         enum intel_engine_id id;
 388
 389         for_each_engine(engine, i915, id)
 390                 __setup_engine_capabilities(engine);
 391 }
 392
 393 /**
 394  * intel_engines_cleanup() - free the resources allocated for Command Streamers
 395  * @i915: the i915 devic
 396  */
 397 void intel_engines_cleanup(struct drm_i915_private *i915)
 398 {
 399         struct intel_engine_cs *engine;
 400         enum intel_engine_id id;
 401
 402         for_each_engine(engine, i915, id) {
 403                 engine->destroy(engine);
 404                 i915->engine[id] = NULL;
 405         }
 406 }
 407
 408 /**
 409  * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
 410  * @i915: the i915 device
 411  *
 412  * Return: non-zero if the initialization failed.
 413  */
 414 int intel_engines_init_mmio(struct drm_i915_private *i915)
 415 {
 416         struct intel_device_info *device_info = mkwrite_device_info(i915);
 417         const unsigned int engine_mask = INTEL_INFO(i915)->engine_mask;
 418         unsigned int mask = 0;
 419         unsigned int i;
 420         int err;
 421
 422         WARN_ON(engine_mask == 0);
 423         WARN_ON(engine_mask &
 424                 GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES));
 425
 426         if (i915_inject_load_failure())
 427                 return -ENODEV;
 428
 429         for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
 430                 if (!HAS_ENGINE(i915, i))
 431                         continue;
 432
 433                 err = intel_engine_setup(i915, i);
 434                 if (err)
 435                         goto cleanup;
 436
 437                 mask |= BIT(i);
 438         }
 439
 440         /*
 441          * Catch failures to update intel_engines table when the new engines
 442          * are added to the driver by a warning and disabling the forgotten
 443          * engines.
 444          */
 445         if (WARN_ON(mask != engine_mask))
 446                 device_info->engine_mask = mask;
 447
 448         /* We always presume we have at least RCS available for later probing */
 449         if (WARN_ON(!HAS_ENGINE(i915, RCS0))) {
 450                 err = -ENODEV;
 451                 goto cleanup;
 452         }
 453
 454         RUNTIME_INFO(i915)->num_engines = hweight32(mask);
 455
 456         i915_check_and_clear_faults(i915);
 457
 458         intel_setup_engine_capabilities(i915);
 459
 460         return 0;
 461
 462 cleanup:
 463         intel_engines_cleanup(i915);
 464         return err;
 465 }
 466
 467 /**
 468  * intel_engines_init() - init the Engine Command Streamers
 469  * @i915: i915 device private
 470  *
 471  * Return: non-zero if the initialization failed.
 472  */
 473 int intel_engines_init(struct drm_i915_private *i915)
 474 {
 475         int (*init)(struct intel_engine_cs *engine);
 476         struct intel_engine_cs *engine;
 477         enum intel_engine_id id;
 478         int err;
 479
 480         if (HAS_EXECLISTS(i915))
 481                 init = intel_execlists_submission_init;
 482         else
 483                 init = intel_ring_submission_init;
 484
 485         for_each_engine(engine, i915, id) {
 486                 err = init(engine);
 487                 if (err)
 488                         goto cleanup;
 489         }
 490
 491         return 0;
 492
 493 cleanup:
 494         intel_engines_cleanup(i915);
 495         return err;
 496 }
 497
 498 static void intel_engine_init_batch_pool(struct intel_engine_cs *engine)
 499 {
 500         i915_gem_batch_pool_init(&engine->batch_pool, engine);
 501 }
 502
 503 void intel_engine_init_execlists(struct intel_engine_cs *engine)
 504 {
 505         struct intel_engine_execlists * const execlists = &engine->execlists;
 506
 507         execlists->port_mask = 1;
 508         GEM_BUG_ON(!is_power_of_2(execlists_num_ports(execlists)));
 509         GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
 510
 511         execlists->queue_priority_hint = INT_MIN;
 512         execlists->queue = RB_ROOT_CACHED;
 513 }
 514
 515 static void cleanup_status_page(struct intel_engine_cs *engine)
 516 {
 517         struct i915_vma *vma;
 518
 519         /* Prevent writes into HWSP after returning the page to the system */
 520         intel_engine_set_hwsp_writemask(engine, ~0u);
 521
 522         vma = fetch_and_zero(&engine->status_page.vma);
 523         if (!vma)
 524                 return;
 525
 526         if (!HWS_NEEDS_PHYSICAL(engine->i915))
 527                 i915_vma_unpin(vma);
 528
 529         i915_gem_object_unpin_map(vma->obj);
 530         i915_gem_object_put(vma->obj);
 531 }
 532
 533 static int pin_ggtt_status_page(struct intel_engine_cs *engine,
 534                                 struct i915_vma *vma)
 535 {
 536         unsigned int flags;
 537
 538         flags = PIN_GLOBAL;
 539         if (!HAS_LLC(engine->i915))
 540                 /*
 541                  * On g33, we cannot place HWS above 256MiB, so
 542                  * restrict its pinning to the low mappable arena.
 543                  * Though this restriction is not documented for
 544                  * gen4, gen5, or byt, they also behave similarly
 545                  * and hang if the HWS is placed at the top of the
 546                  * GTT. To generalise, it appears that all !llc
 547                  * platforms have issues with us placing the HWS
 548                  * above the mappable region (even though we never
 549                  * actually map it).
 550                  */
 551                 flags |= PIN_MAPPABLE;
 552         else
 553                 flags |= PIN_HIGH;
 554
 555         return i915_vma_pin(vma, 0, 0, flags);
 556 }
 557
 558 static int init_status_page(struct intel_engine_cs *engine)
 559 {
 560         struct drm_i915_gem_object *obj;
 561         struct i915_vma *vma;
 562         void *vaddr;
 563         int ret;
 564
 565         /*
 566          * Though the HWS register does support 36bit addresses, historically
 567          * we have had hangs and corruption reported due to wild writes if
 568          * the HWS is placed above 4G. We only allow objects to be allocated
 569          * in GFP_DMA32 for i965, and no earlier physical address users had
 570          * access to more than 4G.
 571          */
 572         obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
 573         if (IS_ERR(obj)) {
 574                 DRM_ERROR("Failed to allocate status page\n");
 575                 return PTR_ERR(obj);
 576         }
 577
 578         i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
 579
 580         vma = i915_vma_instance(obj, &engine->i915->ggtt.vm, NULL);
 581         if (IS_ERR(vma)) {
 582                 ret = PTR_ERR(vma);
 583                 goto err;
 584         }
 585
 586         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
 587         if (IS_ERR(vaddr)) {
 588                 ret = PTR_ERR(vaddr);
 589                 goto err;
 590         }
 591
 592         engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE);
 593         engine->status_page.vma = vma;
 594
 595         if (!HWS_NEEDS_PHYSICAL(engine->i915)) {
 596                 ret = pin_ggtt_status_page(engine, vma);
 597                 if (ret)
 598                         goto err_unpin;
 599         }
 600
 601         return 0;
 602
 603 err_unpin:
 604         i915_gem_object_unpin_map(obj);
 605 err:
 606         i915_gem_object_put(obj);
 607         return ret;
 608 }
 609
 610 static int intel_engine_setup_common(struct intel_engine_cs *engine)
 611 {
 612         int err;
 613
 614         init_llist_head(&engine->barrier_tasks);
 615
 616         err = init_status_page(engine);
 617         if (err)
 618                 return err;
 619
 620         intel_engine_init_active(engine, ENGINE_PHYSICAL);
 621         intel_engine_init_breadcrumbs(engine);
 622         intel_engine_init_execlists(engine);
 623         intel_engine_init_hangcheck(engine);
 624         intel_engine_init_batch_pool(engine);
 625         intel_engine_init_cmd_parser(engine);
 626         intel_engine_init__pm(engine);
 627
 628         /* Use the whole device by default */
 629         engine->sseu =
 630                 intel_sseu_from_device_info(&RUNTIME_INFO(engine->i915)->sseu);
 631
 632         return 0;
 633 }
 634
 635 /**
 636  * intel_engines_setup- setup engine state not requiring hw access
 637  * @i915: Device to setup.
 638  *
 639  * Initializes engine structure members shared between legacy and execlists
 640  * submission modes which do not require hardware access.
 641  *
 642  * Typically done early in the submission mode specific engine setup stage.
 643  */
 644 int intel_engines_setup(struct drm_i915_private *i915)
 645 {
 646         int (*setup)(struct intel_engine_cs *engine);
 647         struct intel_engine_cs *engine;
 648         enum intel_engine_id id;
 649         int err;
 650
 651         if (HAS_EXECLISTS(i915))
 652                 setup = intel_execlists_submission_setup;
 653         else
 654                 setup = intel_ring_submission_setup;
 655
 656         for_each_engine(engine, i915, id) {
 657                 err = intel_engine_setup_common(engine);
 658                 if (err)
 659                         goto cleanup;
 660
 661                 err = setup(engine);
 662                 if (err)
 663                         goto cleanup;
 664
 665                 /* We expect the backend to take control over its state */
 666                 GEM_BUG_ON(engine->destroy == (typeof(engine->destroy))kfree);
 667
 668                 GEM_BUG_ON(!engine->cops);
 669         }
 670
 671         return 0;
 672
 673 cleanup:
 674         intel_engines_cleanup(i915);
 675         return err;
 676 }
 677
 678 void intel_engines_set_scheduler_caps(struct drm_i915_private *i915)
 679 {
 680         static const struct {
 681                 u8 engine;
 682                 u8 sched;
 683         } map[] = {
 684 #define MAP(x, y) { ilog2(I915_ENGINE_HAS_##x), ilog2(I915_SCHEDULER_CAP_##y) }
 685                 MAP(PREEMPTION, PREEMPTION),
 686                 MAP(SEMAPHORES, SEMAPHORES),
 687 #undef MAP
 688         };
 689         struct intel_engine_cs *engine;
 690         enum intel_engine_id id;
 691         u32 enabled, disabled;
 692
 693         enabled = 0;
 694         disabled = 0;
 695         for_each_engine(engine, i915, id) { /* all engines must agree! */
 696                 int i;
 697
 698                 if (engine->schedule)
 699                         enabled |= (I915_SCHEDULER_CAP_ENABLED |
 700                                     I915_SCHEDULER_CAP_PRIORITY);
 701                 else
 702                         disabled |= (I915_SCHEDULER_CAP_ENABLED |
 703                                      I915_SCHEDULER_CAP_PRIORITY);
 704
 705                 for (i = 0; i < ARRAY_SIZE(map); i++) {
 706                         if (engine->flags & BIT(map[i].engine))
 707                                 enabled |= BIT(map[i].sched);
 708                         else
 709                                 disabled |= BIT(map[i].sched);
 710                 }
 711         }
 712
 713         i915->caps.scheduler = enabled & ~disabled;
 714         if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_ENABLED))
 715                 i915->caps.scheduler = 0;
 716 }
 717
 718 struct measure_breadcrumb {
 719         struct i915_request rq;
 720         struct i915_timeline timeline;
 721         struct intel_ring ring;
 722         u32 cs[1024];
 723 };
 724
 725 static int measure_breadcrumb_dw(struct intel_engine_cs *engine)
 726 {
 727         struct measure_breadcrumb *frame;
 728         int dw = -ENOMEM;
 729
 730         GEM_BUG_ON(!engine->i915->gt.scratch);
 731
 732         frame = kzalloc(sizeof(*frame), GFP_KERNEL);
 733         if (!frame)
 734                 return -ENOMEM;
 735
 736         if (i915_timeline_init(engine->i915,
 737                                &frame->timeline,
 738                                engine->status_page.vma))
 739                 goto out_frame;
 740
 741         INIT_LIST_HEAD(&frame->ring.request_list);
 742         frame->ring.timeline = &frame->timeline;
 743         frame->ring.vaddr = frame->cs;
 744         frame->ring.size = sizeof(frame->cs);
 745         frame->ring.effective_size = frame->ring.size;
 746         intel_ring_update_space(&frame->ring);
 747
 748         frame->rq.i915 = engine->i915;
 749         frame->rq.engine = engine;
 750         frame->rq.ring = &frame->ring;
 751         frame->rq.timeline = &frame->timeline;
 752
 753         dw = i915_timeline_pin(&frame->timeline);
 754         if (dw < 0)
 755                 goto out_timeline;
 756
 757         dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs;
 758         GEM_BUG_ON(dw & 1); /* RING_TAIL must be qword aligned */
 759
 760         i915_timeline_unpin(&frame->timeline);
 761
 762 out_timeline:
 763         i915_timeline_fini(&frame->timeline);
 764 out_frame:
 765         kfree(frame);
 766         return dw;
 767 }
 768
 769 static int pin_context(struct i915_gem_context *ctx,
 770                        struct intel_engine_cs *engine,
 771                        struct intel_context **out)
 772 {
 773         struct intel_context *ce;
 774         int err;
 775
 776         ce = i915_gem_context_get_engine(ctx, engine->id);
 777         if (IS_ERR(ce))
 778                 return PTR_ERR(ce);
 779
 780         err = intel_context_pin(ce);
 781         intel_context_put(ce);
 782         if (err)
 783                 return err;
 784
 785         *out = ce;
 786         return 0;
 787 }
 788
 789 void
 790 intel_engine_init_active(struct intel_engine_cs *engine, unsigned int subclass)
 791 {
 792         INIT_LIST_HEAD(&engine->active.requests);
 793
 794         spin_lock_init(&engine->active.lock);
 795         lockdep_set_subclass(&engine->active.lock, subclass);
 796
 797         /*
 798          * Due to an interesting quirk in lockdep's internal debug tracking,
 799          * after setting a subclass we must ensure the lock is used. Otherwise,
 800          * nr_unused_locks is incremented once too often.
 801          */
 802 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 803         local_irq_disable();
 804         lock_map_acquire(&engine->active.lock.dep_map);
 805         lock_map_release(&engine->active.lock.dep_map);
 806         local_irq_enable();
 807 #endif
 808 }
 809
 810 /**
 811  * intel_engines_init_common - initialize cengine state which might require hw access
 812  * @engine: Engine to initialize.
 813  *
 814  * Initializes @engine@ structure members shared between legacy and execlists
 815  * submission modes which do require hardware access.
 816  *
 817  * Typcally done at later stages of submission mode specific engine setup.
 818  *
 819  * Returns zero on success or an error code on failure.
 820  */
 821 int intel_engine_init_common(struct intel_engine_cs *engine)
 822 {
 823         struct drm_i915_private *i915 = engine->i915;
 824         int ret;
 825
 826         /* We may need to do things with the shrinker which
 827          * require us to immediately switch back to the default
 828          * context. This can cause a problem as pinning the
 829          * default context also requires GTT space which may not
 830          * be available. To avoid this we always pin the default
 831          * context.
 832          */
 833         ret = pin_context(i915->kernel_context, engine,
 834                           &engine->kernel_context);
 835         if (ret)
 836                 return ret;
 837
 838         /*
 839          * Similarly the preempt context must always be available so that
 840          * we can interrupt the engine at any time. However, as preemption
 841          * is optional, we allow it to fail.
 842          */
 843         if (i915->preempt_context)
 844                 pin_context(i915->preempt_context, engine,
 845                             &engine->preempt_context);
 846
 847         ret = measure_breadcrumb_dw(engine);
 848         if (ret < 0)
 849                 goto err_unpin;
 850
 851         engine->emit_fini_breadcrumb_dw = ret;
 852
 853         engine->set_default_submission(engine);
 854
 855         return 0;
 856
 857 err_unpin:
 858         if (engine->preempt_context)
 859                 intel_context_unpin(engine->preempt_context);
 860         intel_context_unpin(engine->kernel_context);
 861         return ret;
 862 }
 863
 864 /**
 865  * intel_engines_cleanup_common - cleans up the engine state created by
 866  *                                the common initiailizers.
 867  * @engine: Engine to cleanup.
 868  *
 869  * This cleans up everything created by the common helpers.
 870  */
 871 void intel_engine_cleanup_common(struct intel_engine_cs *engine)
 872 {
 873         GEM_BUG_ON(!list_empty(&engine->active.requests));
 874
 875         cleanup_status_page(engine);
 876
 877         intel_engine_fini_breadcrumbs(engine);
 878         intel_engine_cleanup_cmd_parser(engine);
 879         i915_gem_batch_pool_fini(&engine->batch_pool);
 880
 881         if (engine->default_state)
 882                 i915_gem_object_put(engine->default_state);
 883
 884         if (engine->preempt_context)
 885                 intel_context_unpin(engine->preempt_context);
 886         intel_context_unpin(engine->kernel_context);
 887         GEM_BUG_ON(!llist_empty(&engine->barrier_tasks));
 888
 889         intel_wa_list_free(&engine->ctx_wa_list);
 890         intel_wa_list_free(&engine->wa_list);
 891         intel_wa_list_free(&engine->whitelist);
 892 }
 893
 894 u64 intel_engine_get_active_head(const struct intel_engine_cs *engine)
 895 {
 896         struct drm_i915_private *i915 = engine->i915;
 897
 898         u64 acthd;
 899
 900         if (INTEL_GEN(i915) >= 8)
 901                 acthd = ENGINE_READ64(engine, RING_ACTHD, RING_ACTHD_UDW);
 902         else if (INTEL_GEN(i915) >= 4)
 903                 acthd = ENGINE_READ(engine, RING_ACTHD);
 904         else
 905                 acthd = ENGINE_READ(engine, ACTHD);
 906
 907         return acthd;
 908 }
 909
 910 u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine)
 911 {
 912         u64 bbaddr;
 913
 914         if (INTEL_GEN(engine->i915) >= 8)
 915                 bbaddr = ENGINE_READ64(engine, RING_BBADDR, RING_BBADDR_UDW);
 916         else
 917                 bbaddr = ENGINE_READ(engine, RING_BBADDR);
 918
 919         return bbaddr;
 920 }
 921
 922 int intel_engine_stop_cs(struct intel_engine_cs *engine)
 923 {
 924         struct intel_uncore *uncore = engine->uncore;
 925         const u32 base = engine->mmio_base;
 926         const i915_reg_t mode = RING_MI_MODE(base);
 927         int err;
 928
 929         if (INTEL_GEN(engine->i915) < 3)
 930                 return -ENODEV;
 931
 932         GEM_TRACE("%s\n", engine->name);
 933
 934         intel_uncore_write_fw(uncore, mode, _MASKED_BIT_ENABLE(STOP_RING));
 935
 936         err = 0;
 937         if (__intel_wait_for_register_fw(uncore,
 938                                          mode, MODE_IDLE, MODE_IDLE,
 939                                          1000, 0,
 940                                          NULL)) {
 941                 GEM_TRACE("%s: timed out on STOP_RING -> IDLE\n", engine->name);
 942                 err = -ETIMEDOUT;
 943         }
 944
 945         /* A final mmio read to let GPU writes be hopefully flushed to memory */
 946         intel_uncore_posting_read_fw(uncore, mode);
 947
 948         return err;
 949 }
 950
 951 void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine)
 952 {
 953         GEM_TRACE("%s\n", engine->name);
 954
 955         ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
 956 }
 957
 958 const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
 959 {
 960         switch (type) {
 961         case I915_CACHE_NONE: return " uncached";
 962         case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
 963         case I915_CACHE_L3_LLC: return " L3+LLC";
 964         case I915_CACHE_WT: return " WT";
 965         default: return "";
 966         }
 967 }
 968
 969 u32 intel_calculate_mcr_s_ss_select(struct drm_i915_private *dev_priv)
 970 {
 971         const struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu;
 972         unsigned int slice = fls(sseu->slice_mask) - 1;
 973         unsigned int subslice;
 974         u32 mcr_s_ss_select;
 975
 976         GEM_BUG_ON(slice >= ARRAY_SIZE(sseu->subslice_mask));
 977         subslice = fls(sseu->subslice_mask[slice]);
 978         GEM_BUG_ON(!subslice);
 979         subslice--;
 980
 981         if (IS_GEN(dev_priv, 10))
 982                 mcr_s_ss_select = GEN8_MCR_SLICE(slice) |
 983                                   GEN8_MCR_SUBSLICE(subslice);
 984         else if (INTEL_GEN(dev_priv) >= 11)
 985                 mcr_s_ss_select = GEN11_MCR_SLICE(slice) |
 986                                   GEN11_MCR_SUBSLICE(subslice);
 987         else
 988                 mcr_s_ss_select = 0;
 989
 990         return mcr_s_ss_select;
 991 }
 992
 993 static u32
 994 read_subslice_reg(struct intel_engine_cs *engine, int slice, int subslice,
 995                   i915_reg_t reg)
 996 {
 997         struct drm_i915_private *i915 = engine->i915;
 998         struct intel_uncore *uncore = engine->uncore;
 999         u32 mcr_slice_subslice_mask;
1000         u32 mcr_slice_subslice_select;
1001         u32 default_mcr_s_ss_select;
1002         u32 mcr;
1003         u32 ret;
1004         enum forcewake_domains fw_domains;
1005
1006         if (INTEL_GEN(i915) >= 11) {
1007                 mcr_slice_subslice_mask = GEN11_MCR_SLICE_MASK |
1008                                           GEN11_MCR_SUBSLICE_MASK;
1009                 mcr_slice_subslice_select = GEN11_MCR_SLICE(slice) |
1010                                             GEN11_MCR_SUBSLICE(subslice);
1011         } else {
1012                 mcr_slice_subslice_mask = GEN8_MCR_SLICE_MASK |
1013                                           GEN8_MCR_SUBSLICE_MASK;
1014                 mcr_slice_subslice_select = GEN8_MCR_SLICE(slice) |
1015                                             GEN8_MCR_SUBSLICE(subslice);
1016         }
1017
1018         default_mcr_s_ss_select = intel_calculate_mcr_s_ss_select(i915);
1019
1020         fw_domains = intel_uncore_forcewake_for_reg(uncore, reg,
1021                                                     FW_REG_READ);
1022         fw_domains |= intel_uncore_forcewake_for_reg(uncore,
1023                                                      GEN8_MCR_SELECTOR,
1024                                                      FW_REG_READ | FW_REG_WRITE);
1025
1026         spin_lock_irq(&uncore->lock);
1027         intel_uncore_forcewake_get__locked(uncore, fw_domains);
1028
1029         mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
1030
1031         WARN_ON_ONCE((mcr & mcr_slice_subslice_mask) !=
1032                      default_mcr_s_ss_select);
1033
1034         mcr &= ~mcr_slice_subslice_mask;
1035         mcr |= mcr_slice_subslice_select;
1036         intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
1037
1038         ret = intel_uncore_read_fw(uncore, reg);
1039
1040         mcr &= ~mcr_slice_subslice_mask;
1041         mcr |= default_mcr_s_ss_select;
1042
1043         intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
1044
1045         intel_uncore_forcewake_put__locked(uncore, fw_domains);
1046         spin_unlock_irq(&uncore->lock);
1047
1048         return ret;
1049 }
1050
1051 /* NB: please notice the memset */
1052 void intel_engine_get_instdone(struct intel_engine_cs *engine,
1053                                struct intel_instdone *instdone)
1054 {
1055         struct drm_i915_private *i915 = engine->i915;
1056         struct intel_uncore *uncore = engine->uncore;
1057         u32 mmio_base = engine->mmio_base;
1058         int slice;
1059         int subslice;
1060
1061         memset(instdone, 0, sizeof(*instdone));
1062
1063         switch (INTEL_GEN(i915)) {
1064         default:
1065                 instdone->instdone =
1066                         intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1067
1068                 if (engine->id != RCS0)
1069                         break;
1070
1071                 instdone->slice_common =
1072                         intel_uncore_read(uncore, GEN7_SC_INSTDONE);
1073                 for_each_instdone_slice_subslice(i915, slice, subslice) {
1074                         instdone->sampler[slice][subslice] =
1075                                 read_subslice_reg(engine, slice, subslice,
1076                                                   GEN7_SAMPLER_INSTDONE);
1077                         instdone->row[slice][subslice] =
1078                                 read_subslice_reg(engine, slice, subslice,
1079                                                   GEN7_ROW_INSTDONE);
1080                 }
1081                 break;
1082         case 7:
1083                 instdone->instdone =
1084                         intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1085
1086                 if (engine->id != RCS0)
1087                         break;
1088
1089                 instdone->slice_common =
1090                         intel_uncore_read(uncore, GEN7_SC_INSTDONE);
1091                 instdone->sampler[0][0] =
1092                         intel_uncore_read(uncore, GEN7_SAMPLER_INSTDONE);
1093                 instdone->row[0][0] =
1094                         intel_uncore_read(uncore, GEN7_ROW_INSTDONE);
1095
1096                 break;
1097         case 6:
1098         case 5:
1099         case 4:
1100                 instdone->instdone =
1101                         intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1102                 if (engine->id == RCS0)
1103                         /* HACK: Using the wrong struct member */
1104                         instdone->slice_common =
1105                                 intel_uncore_read(uncore, GEN4_INSTDONE1);
1106                 break;
1107         case 3:
1108         case 2:
1109                 instdone->instdone = intel_uncore_read(uncore, GEN2_INSTDONE);
1110                 break;
1111         }
1112 }
1113
1114 static bool ring_is_idle(struct intel_engine_cs *engine)
1115 {
1116         struct drm_i915_private *dev_priv = engine->i915;
1117         intel_wakeref_t wakeref;
1118         bool idle = true;
1119
1120         if (I915_SELFTEST_ONLY(!engine->mmio_base))
1121                 return true;
1122
1123         /* If the whole device is asleep, the engine must be idle */
1124         wakeref = intel_runtime_pm_get_if_in_use(&dev_priv->runtime_pm);
1125         if (!wakeref)
1126                 return true;
1127
1128         /* First check that no commands are left in the ring */
1129         if ((ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) !=
1130             (ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR))
1131                 idle = false;
1132
1133         /* No bit for gen2, so assume the CS parser is idle */
1134         if (INTEL_GEN(dev_priv) > 2 &&
1135             !(ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE))
1136                 idle = false;
1137
1138         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
1139
1140         return idle;
1141 }
1142
1143 /**
1144  * intel_engine_is_idle() - Report if the engine has finished process all work
1145  * @engine: the intel_engine_cs
1146  *
1147  * Return true if there are no requests pending, nothing left to be submitted
1148  * to hardware, and that the engine is idle.
1149  */
1150 bool intel_engine_is_idle(struct intel_engine_cs *engine)
1151 {
1152         /* More white lies, if wedged, hw state is inconsistent */
1153         if (i915_reset_failed(engine->i915))
1154                 return true;
1155
1156         if (!intel_wakeref_active(&engine->wakeref))
1157                 return true;
1158
1159         /* Waiting to drain ELSP? */
1160         if (READ_ONCE(engine->execlists.active)) {
1161                 struct tasklet_struct *t = &engine->execlists.tasklet;
1162
1163                 synchronize_hardirq(engine->i915->drm.irq);
1164
1165                 local_bh_disable();
1166                 if (tasklet_trylock(t)) {
1167                         /* Must wait for any GPU reset in progress. */
1168                         if (__tasklet_is_enabled(t))
1169                                 t->func(t->data);
1170                         tasklet_unlock(t);
1171                 }
1172                 local_bh_enable();
1173
1174                 /* Otherwise flush the tasklet if it was on another cpu */
1175                 tasklet_unlock_wait(t);
1176
1177                 if (READ_ONCE(engine->execlists.active))
1178                         return false;
1179         }
1180
1181         /* ELSP is empty, but there are ready requests? E.g. after reset */
1182         if (!RB_EMPTY_ROOT(&engine->execlists.queue.rb_root))
1183                 return false;
1184
1185         /* Ring stopped? */
1186         return ring_is_idle(engine);
1187 }
1188
1189 bool intel_engines_are_idle(struct drm_i915_private *i915)
1190 {
1191         struct intel_engine_cs *engine;
1192         enum intel_engine_id id;
1193
1194         /*
1195          * If the driver is wedged, HW state may be very inconsistent and
1196          * report that it is still busy, even though we have stopped using it.
1197          */
1198         if (i915_reset_failed(i915))
1199                 return true;
1200
1201         /* Already parked (and passed an idleness test); must still be idle */
1202         if (!READ_ONCE(i915->gt.awake))
1203                 return true;
1204
1205         for_each_engine(engine, i915, id) {
1206                 if (!intel_engine_is_idle(engine))
1207                         return false;
1208         }
1209
1210         return true;
1211 }
1212
1213 void intel_engines_reset_default_submission(struct drm_i915_private *i915)
1214 {
1215         struct intel_engine_cs *engine;
1216         enum intel_engine_id id;
1217
1218         for_each_engine(engine, i915, id)
1219                 engine->set_default_submission(engine);
1220 }
1221
1222 bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
1223 {
1224         switch (INTEL_GEN(engine->i915)) {
1225         case 2:
1226                 return false; /* uses physical not virtual addresses */
1227         case 3:
1228                 /* maybe only uses physical not virtual addresses */
1229                 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915));
1230         case 6:
1231                 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */
1232         default:
1233                 return true;
1234         }
1235 }
1236
1237 unsigned int intel_engines_has_context_isolation(struct drm_i915_private *i915)
1238 {
1239         struct intel_engine_cs *engine;
1240         enum intel_engine_id id;
1241         unsigned int which;
1242
1243         which = 0;
1244         for_each_engine(engine, i915, id)
1245                 if (engine->default_state)
1246                         which |= BIT(engine->uabi_class);
1247
1248         return which;
1249 }
1250
1251 static int print_sched_attr(struct drm_i915_private *i915,
1252                             const struct i915_sched_attr *attr,
1253                             char *buf, int x, int len)
1254 {
1255         if (attr->priority == I915_PRIORITY_INVALID)
1256                 return x;
1257
1258         x += snprintf(buf + x, len - x,
1259                       " prio=%d", attr->priority);
1260
1261         return x;
1262 }
1263
1264 static void print_request(struct drm_printer *m,
1265                           struct i915_request *rq,
1266                           const char *prefix)
1267 {
1268         const char *name = rq->fence.ops->get_timeline_name(&rq->fence);
1269         char buf[80] = "";
1270         int x = 0;
1271
1272         x = print_sched_attr(rq->i915, &rq->sched.attr, buf, x, sizeof(buf));
1273
1274         drm_printf(m, "%s %llx:%llx%s%s %s @ %dms: %s\n",
1275                    prefix,
1276                    rq->fence.context, rq->fence.seqno,
1277                    i915_request_completed(rq) ? "!" :
1278                    i915_request_started(rq) ? "*" :
1279                    "",
1280                    test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
1281                             &rq->fence.flags) ? "+" :
1282                    test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
1283                             &rq->fence.flags) ? "-" :
1284                    "",
1285                    buf,
1286                    jiffies_to_msecs(jiffies - rq->emitted_jiffies),
1287                    name);
1288 }
1289
1290 static void hexdump(struct drm_printer *m, const void *buf, size_t len)
1291 {
1292         const size_t rowsize = 8 * sizeof(u32);
1293         const void *prev = NULL;
1294         bool skip = false;
1295         size_t pos;
1296
1297         for (pos = 0; pos < len; pos += rowsize) {
1298                 char line[128];
1299
1300                 if (prev && !memcmp(prev, buf + pos, rowsize)) {
1301                         if (!skip) {
1302                                 drm_printf(m, "*\n");
1303                                 skip = true;
1304                         }
1305                         continue;
1306                 }
1307
1308                 WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos,
1309                                                 rowsize, sizeof(u32),
1310                                                 line, sizeof(line),
1311                                                 false) >= sizeof(line));
1312                 drm_printf(m, "[%04zx] %s\n", pos, line);
1313
1314                 prev = buf + pos;
1315                 skip = false;
1316         }
1317 }
1318
1319 static void intel_engine_print_registers(struct intel_engine_cs *engine,
1320                                          struct drm_printer *m)
1321 {
1322         struct drm_i915_private *dev_priv = engine->i915;
1323         const struct intel_engine_execlists * const execlists =
1324                 &engine->execlists;
1325         unsigned long flags;
1326         u64 addr;
1327
1328         if (engine->id == RCS0 && IS_GEN_RANGE(dev_priv, 4, 7))
1329                 drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID));
1330         drm_printf(m, "\tRING_START: 0x%08x\n",
1331                    ENGINE_READ(engine, RING_START));
1332         drm_printf(m, "\tRING_HEAD:  0x%08x\n",
1333                    ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR);
1334         drm_printf(m, "\tRING_TAIL:  0x%08x\n",
1335                    ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR);
1336         drm_printf(m, "\tRING_CTL:   0x%08x%s\n",
1337                    ENGINE_READ(engine, RING_CTL),
1338                    ENGINE_READ(engine, RING_CTL) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
1339         if (INTEL_GEN(engine->i915) > 2) {
1340                 drm_printf(m, "\tRING_MODE:  0x%08x%s\n",
1341                            ENGINE_READ(engine, RING_MI_MODE),
1342                            ENGINE_READ(engine, RING_MI_MODE) & (MODE_IDLE) ? " [idle]" : "");
1343         }
1344
1345         if (INTEL_GEN(dev_priv) >= 6) {
1346                 drm_printf(m, "\tRING_IMR: %08x\n",
1347                            ENGINE_READ(engine, RING_IMR));
1348         }
1349
1350         addr = intel_engine_get_active_head(engine);
1351         drm_printf(m, "\tACTHD:  0x%08x_%08x\n",
1352                    upper_32_bits(addr), lower_32_bits(addr));
1353         addr = intel_engine_get_last_batch_head(engine);
1354         drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
1355                    upper_32_bits(addr), lower_32_bits(addr));
1356         if (INTEL_GEN(dev_priv) >= 8)
1357                 addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW);
1358         else if (INTEL_GEN(dev_priv) >= 4)
1359                 addr = ENGINE_READ(engine, RING_DMA_FADD);
1360         else
1361                 addr = ENGINE_READ(engine, DMA_FADD_I8XX);
1362         drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n",
1363                    upper_32_bits(addr), lower_32_bits(addr));
1364         if (INTEL_GEN(dev_priv) >= 4) {
1365                 drm_printf(m, "\tIPEIR: 0x%08x\n",
1366                            ENGINE_READ(engine, RING_IPEIR));
1367                 drm_printf(m, "\tIPEHR: 0x%08x\n",
1368                            ENGINE_READ(engine, RING_IPEHR));
1369         } else {
1370                 drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, IPEIR));
1371                 drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR));
1372         }
1373
1374         if (HAS_EXECLISTS(dev_priv)) {
1375                 const u32 *hws =
1376                         &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
1377                 const u8 num_entries = execlists->csb_size;
1378                 unsigned int idx;
1379                 u8 read, write;
1380
1381                 drm_printf(m, "\tExeclist status: 0x%08x %08x, entries %u\n",
1382                            ENGINE_READ(engine, RING_EXECLIST_STATUS_LO),
1383                            ENGINE_READ(engine, RING_EXECLIST_STATUS_HI),
1384                            num_entries);
1385
1386                 read = execlists->csb_head;
1387                 write = READ_ONCE(*execlists->csb_write);
1388
1389                 drm_printf(m, "\tExeclist CSB read %d, write %d, tasklet queued? %s (%s)\n",
1390                            read, write,
1391                            yesno(test_bit(TASKLET_STATE_SCHED,
1392                                           &engine->execlists.tasklet.state)),
1393                            enableddisabled(!atomic_read(&engine->execlists.tasklet.count)));
1394                 if (read >= num_entries)
1395                         read = 0;
1396                 if (write >= num_entries)
1397                         write = 0;
1398                 if (read > write)
1399                         write += num_entries;
1400                 while (read < write) {
1401                         idx = ++read % num_entries;
1402                         drm_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n",
1403                                    idx, hws[idx * 2], hws[idx * 2 + 1]);
1404                 }
1405
1406                 spin_lock_irqsave(&engine->active.lock, flags);
1407                 for (idx = 0; idx < execlists_num_ports(execlists); idx++) {
1408                         struct i915_request *rq;
1409                         unsigned int count;
1410                         char hdr[80];
1411
1412                         rq = port_unpack(&execlists->port[idx], &count);
1413                         if (!rq) {
1414                                 drm_printf(m, "\t\tELSP[%d] idle\n", idx);
1415                         } else if (!i915_request_signaled(rq)) {
1416                                 snprintf(hdr, sizeof(hdr),
1417                                          "\t\tELSP[%d] count=%d, ring:{start:%08x, hwsp:%08x, seqno:%08x}, rq: ",
1418                                          idx, count,
1419                                          i915_ggtt_offset(rq->ring->vma),
1420                                          rq->timeline->hwsp_offset,
1421                                          hwsp_seqno(rq));
1422                                 print_request(m, rq, hdr);
1423                         } else {
1424                                 print_request(m, rq, "\t\tELSP[%d] rq: ");
1425                         }
1426                 }
1427                 drm_printf(m, "\t\tHW active? 0x%x\n", execlists->active);
1428                 spin_unlock_irqrestore(&engine->active.lock, flags);
1429         } else if (INTEL_GEN(dev_priv) > 6) {
1430                 drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
1431                            ENGINE_READ(engine, RING_PP_DIR_BASE));
1432                 drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
1433                            ENGINE_READ(engine, RING_PP_DIR_BASE_READ));
1434                 drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
1435                            ENGINE_READ(engine, RING_PP_DIR_DCLV));
1436         }
1437 }
1438
1439 static void print_request_ring(struct drm_printer *m, struct i915_request *rq)
1440 {
1441         void *ring;
1442         int size;
1443
1444         drm_printf(m,
1445                    "[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]:\n",
1446                    rq->head, rq->postfix, rq->tail,
1447                    rq->batch ? upper_32_bits(rq->batch->node.start) : ~0u,
1448                    rq->batch ? lower_32_bits(rq->batch->node.start) : ~0u);
1449
1450         size = rq->tail - rq->head;
1451         if (rq->tail < rq->head)
1452                 size += rq->ring->size;
1453
1454         ring = kmalloc(size, GFP_ATOMIC);
1455         if (ring) {
1456                 const void *vaddr = rq->ring->vaddr;
1457                 unsigned int head = rq->head;
1458                 unsigned int len = 0;
1459
1460                 if (rq->tail < head) {
1461                         len = rq->ring->size - head;
1462                         memcpy(ring, vaddr + head, len);
1463                         head = 0;
1464                 }
1465                 memcpy(ring + len, vaddr + head, size - len);
1466
1467                 hexdump(m, ring, size);
1468                 kfree(ring);
1469         }
1470 }
1471
1472 void intel_engine_dump(struct intel_engine_cs *engine,
1473                        struct drm_printer *m,
1474                        const char *header, ...)
1475 {
1476         struct i915_gpu_error * const error = &engine->i915->gpu_error;
1477         struct i915_request *rq;
1478         intel_wakeref_t wakeref;
1479         unsigned long flags;
1480
1481         if (header) {
1482                 va_list ap;
1483
1484                 va_start(ap, header);
1485                 drm_vprintf(m, header, &ap);
1486                 va_end(ap);
1487         }
1488
1489         if (i915_reset_failed(engine->i915))
1490                 drm_printf(m, "*** WEDGED ***\n");
1491
1492         drm_printf(m, "\tAwake? %d\n", atomic_read(&engine->wakeref.count));
1493         drm_printf(m, "\tHangcheck: %d ms ago\n",
1494                    jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp));
1495         drm_printf(m, "\tReset count: %d (global %d)\n",
1496                    i915_reset_engine_count(error, engine),
1497                    i915_reset_count(error));
1498
1499         drm_printf(m, "\tRequests:\n");
1500
1501         spin_lock_irqsave(&engine->active.lock, flags);
1502         rq = intel_engine_find_active_request(engine);
1503         if (rq) {
1504                 print_request(m, rq, "\t\tactive ");
1505
1506                 drm_printf(m, "\t\tring->start:  0x%08x\n",
1507                            i915_ggtt_offset(rq->ring->vma));
1508                 drm_printf(m, "\t\tring->head:   0x%08x\n",
1509                            rq->ring->head);
1510                 drm_printf(m, "\t\tring->tail:   0x%08x\n",
1511                            rq->ring->tail);
1512                 drm_printf(m, "\t\tring->emit:   0x%08x\n",
1513                            rq->ring->emit);
1514                 drm_printf(m, "\t\tring->space:  0x%08x\n",
1515                            rq->ring->space);
1516                 drm_printf(m, "\t\tring->hwsp:   0x%08x\n",
1517                            rq->timeline->hwsp_offset);
1518
1519                 print_request_ring(m, rq);
1520         }
1521         spin_unlock_irqrestore(&engine->active.lock, flags);
1522
1523         wakeref = intel_runtime_pm_get_if_in_use(&engine->i915->runtime_pm);
1524         if (wakeref) {
1525                 intel_engine_print_registers(engine, m);
1526                 intel_runtime_pm_put(&engine->i915->runtime_pm, wakeref);
1527         } else {
1528                 drm_printf(m, "\tDevice is asleep; skipping register dump\n");
1529         }
1530
1531         intel_execlists_show_requests(engine, m, print_request, 8);
1532
1533         drm_printf(m, "HWSP:\n");
1534         hexdump(m, engine->status_page.addr, PAGE_SIZE);
1535
1536         drm_printf(m, "Idle? %s\n", yesno(intel_engine_is_idle(engine)));
1537
1538         intel_engine_print_breadcrumbs(engine, m);
1539 }
1540
1541 static u8 user_class_map[] = {
1542         [I915_ENGINE_CLASS_RENDER] = RENDER_CLASS,
1543         [I915_ENGINE_CLASS_COPY] = COPY_ENGINE_CLASS,
1544         [I915_ENGINE_CLASS_VIDEO] = VIDEO_DECODE_CLASS,
1545         [I915_ENGINE_CLASS_VIDEO_ENHANCE] = VIDEO_ENHANCEMENT_CLASS,
1546 };
1547
1548 struct intel_engine_cs *
1549 intel_engine_lookup_user(struct drm_i915_private *i915, u8 class, u8 instance)
1550 {
1551         if (class >= ARRAY_SIZE(user_class_map))
1552                 return NULL;
1553
1554         class = user_class_map[class];
1555
1556         GEM_BUG_ON(class > MAX_ENGINE_CLASS);
1557
1558         if (instance > MAX_ENGINE_INSTANCE)
1559                 return NULL;
1560
1561         return i915->engine_class[class][instance];
1562 }
1563
1564 /**
1565  * intel_enable_engine_stats() - Enable engine busy tracking on engine
1566  * @engine: engine to enable stats collection
1567  *
1568  * Start collecting the engine busyness data for @engine.
1569  *
1570  * Returns 0 on success or a negative error code.
1571  */
1572 int intel_enable_engine_stats(struct intel_engine_cs *engine)
1573 {
1574         struct intel_engine_execlists *execlists = &engine->execlists;
1575         unsigned long flags;
1576         int err = 0;
1577
1578         if (!intel_engine_supports_stats(engine))
1579                 return -ENODEV;
1580
1581         spin_lock_irqsave(&engine->active.lock, flags);
1582         write_seqlock(&engine->stats.lock);
1583
1584         if (unlikely(engine->stats.enabled == ~0)) {
1585                 err = -EBUSY;
1586                 goto unlock;
1587         }
1588
1589         if (engine->stats.enabled++ == 0) {
1590                 const struct execlist_port *port = execlists->port;
1591                 unsigned int num_ports = execlists_num_ports(execlists);
1592
1593                 engine->stats.enabled_at = ktime_get();
1594
1595                 /* XXX submission method oblivious? */
1596                 while (num_ports-- && port_isset(port)) {
1597                         engine->stats.active++;
1598                         port++;
1599                 }
1600
1601                 if (engine->stats.active)
1602                         engine->stats.start = engine->stats.enabled_at;
1603         }
1604
1605 unlock:
1606         write_sequnlock(&engine->stats.lock);
1607         spin_unlock_irqrestore(&engine->active.lock, flags);
1608
1609         return err;
1610 }
1611
1612 static ktime_t __intel_engine_get_busy_time(struct intel_engine_cs *engine)
1613 {
1614         ktime_t total = engine->stats.total;
1615
1616         /*
1617          * If the engine is executing something at the moment
1618          * add it to the total.
1619          */
1620         if (engine->stats.active)
1621                 total = ktime_add(total,
1622                                   ktime_sub(ktime_get(), engine->stats.start));
1623
1624         return total;
1625 }
1626
1627 /**
1628  * intel_engine_get_busy_time() - Return current accumulated engine busyness
1629  * @engine: engine to report on
1630  *
1631  * Returns accumulated time @engine was busy since engine stats were enabled.
1632  */
1633 ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine)
1634 {
1635         unsigned int seq;
1636         ktime_t total;
1637
1638         do {
1639                 seq = read_seqbegin(&engine->stats.lock);
1640                 total = __intel_engine_get_busy_time(engine);
1641         } while (read_seqretry(&engine->stats.lock, seq));
1642
1643         return total;
1644 }
1645
1646 /**
1647  * intel_disable_engine_stats() - Disable engine busy tracking on engine
1648  * @engine: engine to disable stats collection
1649  *
1650  * Stops collecting the engine busyness data for @engine.
1651  */
1652 void intel_disable_engine_stats(struct intel_engine_cs *engine)
1653 {
1654         unsigned long flags;
1655
1656         if (!intel_engine_supports_stats(engine))
1657                 return;
1658
1659         write_seqlock_irqsave(&engine->stats.lock, flags);
1660         WARN_ON_ONCE(engine->stats.enabled == 0);
1661         if (--engine->stats.enabled == 0) {
1662                 engine->stats.total = __intel_engine_get_busy_time(engine);
1663                 engine->stats.active = 0;
1664         }
1665         write_sequnlock_irqrestore(&engine->stats.lock, flags);
1666 }
1667
1668 static bool match_ring(struct i915_request *rq)
1669 {
1670         u32 ring = ENGINE_READ(rq->engine, RING_START);
1671
1672         return ring == i915_ggtt_offset(rq->ring->vma);
1673 }
1674
1675 struct i915_request *
1676 intel_engine_find_active_request(struct intel_engine_cs *engine)
1677 {
1678         struct i915_request *request, *active = NULL;
1679
1680         /*
1681          * We are called by the error capture, reset and to dump engine
1682          * state at random points in time. In particular, note that neither is
1683          * crucially ordered with an interrupt. After a hang, the GPU is dead
1684          * and we assume that no more writes can happen (we waited long enough
1685          * for all writes that were in transaction to be flushed) - adding an
1686          * extra delay for a recent interrupt is pointless. Hence, we do
1687          * not need an engine->irq_seqno_barrier() before the seqno reads.
1688          * At all other times, we must assume the GPU is still running, but
1689          * we only care about the snapshot of this moment.
1690          */
1691         lockdep_assert_held(&engine->active.lock);
1692         list_for_each_entry(request, &engine->active.requests, sched.link) {
1693                 if (i915_request_completed(request))
1694                         continue;
1695
1696                 if (!i915_request_started(request))
1697                         continue;
1698
1699                 /* More than one preemptible request may match! */
1700                 if (!match_ring(request))
1701                         continue;
1702
1703                 active = request;
1704                 break;
1705         }
1706
1707         return active;
1708 }
1709
1710 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1711 #include "selftest_engine_cs.c"
1712 #endif