drivers/gpu/drm/i915/gt/intel_gt_irq.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2019 Intel Corporation
   5  */
   6
   7 #include <linux/sched/clock.h>
   8
   9 #include "i915_drv.h"
  10 #include "i915_irq.h"
  11 #include "intel_gt.h"
  12 #include "intel_gt_irq.h"
  13 #include "intel_uncore.h"
  14
  15 static void guc_irq_handler(struct intel_guc *guc, u16 iir)
  16 {
  17         if (iir & GUC_INTR_GUC2HOST)
  18                 intel_guc_to_host_event_handler(guc);
  19 }
  20
  21 static void
  22 cs_irq_handler(struct intel_engine_cs *engine, u32 iir)
  23 {
  24         bool tasklet = false;
  25
  26         if (iir & GT_CONTEXT_SWITCH_INTERRUPT)
  27                 tasklet = true;
  28
  29         if (iir & GT_RENDER_USER_INTERRUPT) {
  30                 intel_engine_breadcrumbs_irq(engine);
  31                 tasklet |= intel_engine_needs_breadcrumb_tasklet(engine);
  32         }
  33
  34         if (tasklet)
  35                 tasklet_hi_schedule(&engine->execlists.tasklet);
  36 }
  37
  38 static u32
  39 gen11_gt_engine_identity(struct intel_gt *gt,
  40                          const unsigned int bank, const unsigned int bit)
  41 {
  42         void __iomem * const regs = gt->uncore->regs;
  43         u32 timeout_ts;
  44         u32 ident;
  45
  46         lockdep_assert_held(&gt->irq_lock);
  47
  48         raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
  49
  50         /*
  51          * NB: Specs do not specify how long to spin wait,
  52          * so we do ~100us as an educated guess.
  53          */
  54         timeout_ts = (local_clock() >> 10) + 100;
  55         do {
  56                 ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank));
  57         } while (!(ident & GEN11_INTR_DATA_VALID) &&
  58                  !time_after32(local_clock() >> 10, timeout_ts));
  59
  60         if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) {
  61                 DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
  62                           bank, bit, ident);
  63                 return 0;
  64         }
  65
  66         raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
  67                       GEN11_INTR_DATA_VALID);
  68
  69         return ident;
  70 }
  71
  72 static void
  73 gen11_other_irq_handler(struct intel_gt *gt, const u8 instance,
  74                         const u16 iir)
  75 {
  76         if (instance == OTHER_GUC_INSTANCE)
  77                 return guc_irq_handler(&gt->uc.guc, iir);
  78
  79         if (instance == OTHER_GTPM_INSTANCE)
  80                 return gen11_rps_irq_handler(gt, iir);
  81
  82         WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
  83                   instance, iir);
  84 }
  85
  86 static void
  87 gen11_engine_irq_handler(struct intel_gt *gt, const u8 class,
  88                          const u8 instance, const u16 iir)
  89 {
  90         struct intel_engine_cs *engine;
  91
  92         if (instance <= MAX_ENGINE_INSTANCE)
  93                 engine = gt->engine_class[class][instance];
  94         else
  95                 engine = NULL;
  96
  97         if (likely(engine))
  98                 return cs_irq_handler(engine, iir);
  99
 100         WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
 101                   class, instance);
 102 }
 103
 104 static void
 105 gen11_gt_identity_handler(struct intel_gt *gt, const u32 identity)
 106 {
 107         const u8 class = GEN11_INTR_ENGINE_CLASS(identity);
 108         const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity);
 109         const u16 intr = GEN11_INTR_ENGINE_INTR(identity);
 110
 111         if (unlikely(!intr))
 112                 return;
 113
 114         if (class <= COPY_ENGINE_CLASS)
 115                 return gen11_engine_irq_handler(gt, class, instance, intr);
 116
 117         if (class == OTHER_CLASS)
 118                 return gen11_other_irq_handler(gt, instance, intr);
 119
 120         WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
 121                   class, instance, intr);
 122 }
 123
 124 static void
 125 gen11_gt_bank_handler(struct intel_gt *gt, const unsigned int bank)
 126 {
 127         void __iomem * const regs = gt->uncore->regs;
 128         unsigned long intr_dw;
 129         unsigned int bit;
 130
 131         lockdep_assert_held(&gt->irq_lock);
 132
 133         intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
 134
 135         for_each_set_bit(bit, &intr_dw, 32) {
 136                 const u32 ident = gen11_gt_engine_identity(gt, bank, bit);
 137
 138                 gen11_gt_identity_handler(gt, ident);
 139         }
 140
 141         /* Clear must be after shared has been served for engine */
 142         raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
 143 }
 144
 145 void gen11_gt_irq_handler(struct intel_gt *gt, const u32 master_ctl)
 146 {
 147         unsigned int bank;
 148
 149         spin_lock(&gt->irq_lock);
 150
 151         for (bank = 0; bank < 2; bank++) {
 152                 if (master_ctl & GEN11_GT_DW_IRQ(bank))
 153                         gen11_gt_bank_handler(gt, bank);
 154         }
 155
 156         spin_unlock(&gt->irq_lock);
 157 }
 158
 159 bool gen11_gt_reset_one_iir(struct intel_gt *gt,
 160                             const unsigned int bank, const unsigned int bit)
 161 {
 162         void __iomem * const regs = gt->uncore->regs;
 163         u32 dw;
 164
 165         lockdep_assert_held(&gt->irq_lock);
 166
 167         dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
 168         if (dw & BIT(bit)) {
 169                 /*
 170                  * According to the BSpec, DW_IIR bits cannot be cleared without
 171                  * first servicing the Selector & Shared IIR registers.
 172                  */
 173                 gen11_gt_engine_identity(gt, bank, bit);
 174
 175                 /*
 176                  * We locked GT INT DW by reading it. If we want to (try
 177                  * to) recover from this successfully, we need to clear
 178                  * our bit, otherwise we are locking the register for
 179                  * everybody.
 180                  */
 181                 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit));
 182
 183                 return true;
 184         }
 185
 186         return false;
 187 }
 188
 189 void gen11_gt_irq_reset(struct intel_gt *gt)
 190 {
 191         struct intel_uncore *uncore = gt->uncore;
 192
 193         /* Disable RCS, BCS, VCS and VECS class engines. */
 194         intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, 0);
 195         intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE,    0);
 196
 197         /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
 198         intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK,   ~0);
 199         intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK,    ~0);
 200         intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK,   ~0);
 201         intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK,   ~0);
 202         intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~0);
 203
 204         intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
 205         intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
 206         intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
 207         intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK,  ~0);
 208 }
 209
 210 void gen11_gt_irq_postinstall(struct intel_gt *gt)
 211 {
 212         const u32 irqs = GT_RENDER_USER_INTERRUPT | GT_CONTEXT_SWITCH_INTERRUPT;
 213         struct intel_uncore *uncore = gt->uncore;
 214         const u32 dmask = irqs << 16 | irqs;
 215         const u32 smask = irqs << 16;
 216
 217         BUILD_BUG_ON(irqs & 0xffff0000);
 218
 219         /* Enable RCS, BCS, VCS and VECS class interrupts. */
 220         intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, dmask);
 221         intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask);
 222
 223         /* Unmask irqs on RCS, BCS, VCS and VECS engines. */
 224         intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask);
 225         intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~smask);
 226         intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~dmask);
 227         intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~dmask);
 228         intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask);
 229
 230         /*
 231          * RPS interrupts will get enabled/disabled on demand when RPS itself
 232          * is enabled/disabled.
 233          */
 234         gt->pm_ier = 0x0;
 235         gt->pm_imr = ~gt->pm_ier;
 236         intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
 237         intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
 238
 239         /* Same thing for GuC interrupts */
 240         intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
 241         intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK,  ~0);
 242 }
 243
 244 void gen5_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
 245 {
 246         if (gt_iir & GT_RENDER_USER_INTERRUPT)
 247                 intel_engine_breadcrumbs_irq(gt->engine_class[RENDER_CLASS][0]);
 248         if (gt_iir & ILK_BSD_USER_INTERRUPT)
 249                 intel_engine_breadcrumbs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0]);
 250 }
 251
 252 static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir)
 253 {
 254         if (!HAS_L3_DPF(gt->i915))
 255                 return;
 256
 257         spin_lock(&gt->irq_lock);
 258         gen5_gt_disable_irq(gt, GT_PARITY_ERROR(gt->i915));
 259         spin_unlock(&gt->irq_lock);
 260
 261         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
 262                 gt->i915->l3_parity.which_slice |= 1 << 1;
 263
 264         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
 265                 gt->i915->l3_parity.which_slice |= 1 << 0;
 266
 267         schedule_work(&gt->i915->l3_parity.error_work);
 268 }
 269
 270 void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
 271 {
 272         if (gt_iir & GT_RENDER_USER_INTERRUPT)
 273                 intel_engine_breadcrumbs_irq(gt->engine_class[RENDER_CLASS][0]);
 274         if (gt_iir & GT_BSD_USER_INTERRUPT)
 275                 intel_engine_breadcrumbs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0]);
 276         if (gt_iir & GT_BLT_USER_INTERRUPT)
 277                 intel_engine_breadcrumbs_irq(gt->engine_class[COPY_ENGINE_CLASS][0]);
 278
 279         if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
 280                       GT_BSD_CS_ERROR_INTERRUPT |
 281                       GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
 282                 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
 283
 284         if (gt_iir & GT_PARITY_ERROR(gt->i915))
 285                 gen7_parity_error_irq_handler(gt, gt_iir);
 286 }
 287
 288 void gen8_gt_irq_ack(struct intel_gt *gt, u32 master_ctl, u32 gt_iir[4])
 289 {
 290         void __iomem * const regs = gt->uncore->regs;
 291
 292         if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
 293                 gt_iir[0] = raw_reg_read(regs, GEN8_GT_IIR(0));
 294                 if (likely(gt_iir[0]))
 295                         raw_reg_write(regs, GEN8_GT_IIR(0), gt_iir[0]);
 296         }
 297
 298         if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
 299                 gt_iir[1] = raw_reg_read(regs, GEN8_GT_IIR(1));
 300                 if (likely(gt_iir[1]))
 301                         raw_reg_write(regs, GEN8_GT_IIR(1), gt_iir[1]);
 302         }
 303
 304         if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
 305                 gt_iir[2] = raw_reg_read(regs, GEN8_GT_IIR(2));
 306                 if (likely(gt_iir[2]))
 307                         raw_reg_write(regs, GEN8_GT_IIR(2), gt_iir[2]);
 308         }
 309
 310         if (master_ctl & GEN8_GT_VECS_IRQ) {
 311                 gt_iir[3] = raw_reg_read(regs, GEN8_GT_IIR(3));
 312                 if (likely(gt_iir[3]))
 313                         raw_reg_write(regs, GEN8_GT_IIR(3), gt_iir[3]);
 314         }
 315 }
 316
 317 void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl, u32 gt_iir[4])
 318 {
 319         if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
 320                 cs_irq_handler(gt->engine_class[RENDER_CLASS][0],
 321                                gt_iir[0] >> GEN8_RCS_IRQ_SHIFT);
 322                 cs_irq_handler(gt->engine_class[COPY_ENGINE_CLASS][0],
 323                                gt_iir[0] >> GEN8_BCS_IRQ_SHIFT);
 324         }
 325
 326         if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
 327                 cs_irq_handler(gt->engine_class[VIDEO_DECODE_CLASS][0],
 328                                gt_iir[1] >> GEN8_VCS0_IRQ_SHIFT);
 329                 cs_irq_handler(gt->engine_class[VIDEO_DECODE_CLASS][1],
 330                                gt_iir[1] >> GEN8_VCS1_IRQ_SHIFT);
 331         }
 332
 333         if (master_ctl & GEN8_GT_VECS_IRQ) {
 334                 cs_irq_handler(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0],
 335                                gt_iir[3] >> GEN8_VECS_IRQ_SHIFT);
 336         }
 337
 338         if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
 339                 gen6_rps_irq_handler(gt->i915, gt_iir[2]);
 340                 guc_irq_handler(&gt->uc.guc, gt_iir[2] >> 16);
 341         }
 342 }
 343
 344 void gen8_gt_irq_reset(struct intel_gt *gt)
 345 {
 346         struct intel_uncore *uncore = gt->uncore;
 347
 348         GEN8_IRQ_RESET_NDX(uncore, GT, 0);
 349         GEN8_IRQ_RESET_NDX(uncore, GT, 1);
 350         GEN8_IRQ_RESET_NDX(uncore, GT, 2);
 351         GEN8_IRQ_RESET_NDX(uncore, GT, 3);
 352 }
 353
 354 void gen8_gt_irq_postinstall(struct intel_gt *gt)
 355 {
 356         struct intel_uncore *uncore = gt->uncore;
 357
 358         /* These are interrupts we'll toggle with the ring mask register */
 359         u32 gt_interrupts[] = {
 360                 (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
 361                  GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
 362                  GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
 363                  GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT),
 364
 365                 (GT_RENDER_USER_INTERRUPT << GEN8_VCS0_IRQ_SHIFT |
 366                  GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS0_IRQ_SHIFT |
 367                  GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
 368                  GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT),
 369
 370                 0,
 371
 372                 (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
 373                  GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT)
 374         };
 375
 376         gt->pm_ier = 0x0;
 377         gt->pm_imr = ~gt->pm_ier;
 378         GEN8_IRQ_INIT_NDX(uncore, GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
 379         GEN8_IRQ_INIT_NDX(uncore, GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
 380         /*
 381          * RPS interrupts will get enabled/disabled on demand when RPS itself
 382          * is enabled/disabled. Same wil be the case for GuC interrupts.
 383          */
 384         GEN8_IRQ_INIT_NDX(uncore, GT, 2, gt->pm_imr, gt->pm_ier);
 385         GEN8_IRQ_INIT_NDX(uncore, GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
 386 }
 387
 388 static void gen5_gt_update_irq(struct intel_gt *gt,
 389                                u32 interrupt_mask,
 390                                u32 enabled_irq_mask)
 391 {
 392         lockdep_assert_held(&gt->irq_lock);
 393
 394         GEM_BUG_ON(enabled_irq_mask & ~interrupt_mask);
 395
 396         gt->gt_imr &= ~interrupt_mask;
 397         gt->gt_imr |= (~enabled_irq_mask & interrupt_mask);
 398         intel_uncore_write(gt->uncore, GTIMR, gt->gt_imr);
 399 }
 400
 401 void gen5_gt_enable_irq(struct intel_gt *gt, u32 mask)
 402 {
 403         gen5_gt_update_irq(gt, mask, mask);
 404         intel_uncore_posting_read_fw(gt->uncore, GTIMR);
 405 }
 406
 407 void gen5_gt_disable_irq(struct intel_gt *gt, u32 mask)
 408 {
 409         gen5_gt_update_irq(gt, mask, 0);
 410 }
 411
 412 void gen5_gt_irq_reset(struct intel_gt *gt)
 413 {
 414         struct intel_uncore *uncore = gt->uncore;
 415
 416         GEN3_IRQ_RESET(uncore, GT);
 417         if (INTEL_GEN(gt->i915) >= 6)
 418                 GEN3_IRQ_RESET(uncore, GEN6_PM);
 419 }
 420
 421 void gen5_gt_irq_postinstall(struct intel_gt *gt)
 422 {
 423         struct intel_uncore *uncore = gt->uncore;
 424         u32 pm_irqs = 0;
 425         u32 gt_irqs = 0;
 426
 427         gt->gt_imr = ~0;
 428         if (HAS_L3_DPF(gt->i915)) {
 429                 /* L3 parity interrupt is always unmasked. */
 430                 gt->gt_imr = ~GT_PARITY_ERROR(gt->i915);
 431                 gt_irqs |= GT_PARITY_ERROR(gt->i915);
 432         }
 433
 434         gt_irqs |= GT_RENDER_USER_INTERRUPT;
 435         if (IS_GEN(gt->i915, 5))
 436                 gt_irqs |= ILK_BSD_USER_INTERRUPT;
 437         else
 438                 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
 439
 440         GEN3_IRQ_INIT(uncore, GT, gt->gt_imr, gt_irqs);
 441
 442         if (INTEL_GEN(gt->i915) >= 6) {
 443                 /*
 444                  * RPS interrupts will get enabled/disabled on demand when RPS
 445                  * itself is enabled/disabled.
 446                  */
 447                 if (HAS_ENGINE(gt->i915, VECS0)) {
 448                         pm_irqs |= PM_VEBOX_USER_INTERRUPT;
 449                         gt->pm_ier |= PM_VEBOX_USER_INTERRUPT;
 450                 }
 451
 452                 gt->pm_imr = 0xffffffff;
 453                 GEN3_IRQ_INIT(uncore, GEN6_PM, gt->pm_imr, pm_irqs);
 454         }
 455 }