drivers/gpu/drm/i915/i915_pmu.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2017-2018 Intel Corporation
   5  */
   6
   7 #include <linux/irq.h>
   8 #include <linux/pm_runtime.h>
   9
  10 #include "gt/intel_engine.h"
  11
  12 #include "i915_drv.h"
  13 #include "i915_pmu.h"
  14 #include "intel_pm.h"
  15
  16 /* Frequency for the sampling timer for events which need it. */
  17 #define FREQUENCY 200
  18 #define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY)
  19
  20 #define ENGINE_SAMPLE_MASK \
  21         (BIT(I915_SAMPLE_BUSY) | \
  22          BIT(I915_SAMPLE_WAIT) | \
  23          BIT(I915_SAMPLE_SEMA))
  24
  25 #define ENGINE_SAMPLE_BITS (1 << I915_PMU_SAMPLE_BITS)
  26
  27 static cpumask_t i915_pmu_cpumask;
  28
  29 static u8 engine_config_sample(u64 config)
  30 {
  31         return config & I915_PMU_SAMPLE_MASK;
  32 }
  33
  34 static u8 engine_event_sample(struct perf_event *event)
  35 {
  36         return engine_config_sample(event->attr.config);
  37 }
  38
  39 static u8 engine_event_class(struct perf_event *event)
  40 {
  41         return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff;
  42 }
  43
  44 static u8 engine_event_instance(struct perf_event *event)
  45 {
  46         return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff;
  47 }
  48
  49 static bool is_engine_config(u64 config)
  50 {
  51         return config < __I915_PMU_OTHER(0);
  52 }
  53
  54 static unsigned int config_enabled_bit(u64 config)
  55 {
  56         if (is_engine_config(config))
  57                 return engine_config_sample(config);
  58         else
  59                 return ENGINE_SAMPLE_BITS + (config - __I915_PMU_OTHER(0));
  60 }
  61
  62 static u64 config_enabled_mask(u64 config)
  63 {
  64         return BIT_ULL(config_enabled_bit(config));
  65 }
  66
  67 static bool is_engine_event(struct perf_event *event)
  68 {
  69         return is_engine_config(event->attr.config);
  70 }
  71
  72 static unsigned int event_enabled_bit(struct perf_event *event)
  73 {
  74         return config_enabled_bit(event->attr.config);
  75 }
  76
  77 static bool pmu_needs_timer(struct drm_i915_private *i915, bool gpu_active)
  78 {
  79         u64 enable;
  80
  81         /*
  82          * Only some counters need the sampling timer.
  83          *
  84          * We start with a bitmask of all currently enabled events.
  85          */
  86         enable = i915->pmu.enable;
  87
  88         /*
  89          * Mask out all the ones which do not need the timer, or in
  90          * other words keep all the ones that could need the timer.
  91          */
  92         enable &= config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY) |
  93                   config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY) |
  94                   ENGINE_SAMPLE_MASK;
  95
  96         /*
  97          * When the GPU is idle per-engine counters do not need to be
  98          * running so clear those bits out.
  99          */
 100         if (!gpu_active)
 101                 enable &= ~ENGINE_SAMPLE_MASK;
 102         /*
 103          * Also there is software busyness tracking available we do not
 104          * need the timer for I915_SAMPLE_BUSY counter.
 105          *
 106          * Use RCS as proxy for all engines.
 107          */
 108         else if (intel_engine_supports_stats(i915->engine[RCS0]))
 109                 enable &= ~BIT(I915_SAMPLE_BUSY);
 110
 111         /*
 112          * If some bits remain it means we need the sampling timer running.
 113          */
 114         return enable;
 115 }
 116
 117 void i915_pmu_gt_parked(struct drm_i915_private *i915)
 118 {
 119         if (!i915->pmu.base.event_init)
 120                 return;
 121
 122         spin_lock_irq(&i915->pmu.lock);
 123         /*
 124          * Signal sampling timer to stop if only engine events are enabled and
 125          * GPU went idle.
 126          */
 127         i915->pmu.timer_enabled = pmu_needs_timer(i915, false);
 128         spin_unlock_irq(&i915->pmu.lock);
 129 }
 130
 131 static void __i915_pmu_maybe_start_timer(struct drm_i915_private *i915)
 132 {
 133         if (!i915->pmu.timer_enabled && pmu_needs_timer(i915, true)) {
 134                 i915->pmu.timer_enabled = true;
 135                 i915->pmu.timer_last = ktime_get();
 136                 hrtimer_start_range_ns(&i915->pmu.timer,
 137                                        ns_to_ktime(PERIOD), 0,
 138                                        HRTIMER_MODE_REL_PINNED);
 139         }
 140 }
 141
 142 void i915_pmu_gt_unparked(struct drm_i915_private *i915)
 143 {
 144         if (!i915->pmu.base.event_init)
 145                 return;
 146
 147         spin_lock_irq(&i915->pmu.lock);
 148         /*
 149          * Re-enable sampling timer when GPU goes active.
 150          */
 151         __i915_pmu_maybe_start_timer(i915);
 152         spin_unlock_irq(&i915->pmu.lock);
 153 }
 154
 155 static void
 156 add_sample(struct i915_pmu_sample *sample, u32 val)
 157 {
 158         sample->cur += val;
 159 }
 160
 161 static void
 162 engines_sample(struct drm_i915_private *dev_priv, unsigned int period_ns)
 163 {
 164         struct intel_engine_cs *engine;
 165         enum intel_engine_id id;
 166         intel_wakeref_t wakeref;
 167         unsigned long flags;
 168
 169         if ((dev_priv->pmu.enable & ENGINE_SAMPLE_MASK) == 0)
 170                 return;
 171
 172         wakeref = 0;
 173         if (READ_ONCE(dev_priv->gt.awake))
 174                 wakeref = intel_runtime_pm_get_if_in_use(&dev_priv->runtime_pm);
 175         if (!wakeref)
 176                 return;
 177
 178         spin_lock_irqsave(&dev_priv->uncore.lock, flags);
 179         for_each_engine(engine, dev_priv, id) {
 180                 struct intel_engine_pmu *pmu = &engine->pmu;
 181                 bool busy;
 182                 u32 val;
 183
 184                 val = I915_READ_FW(RING_CTL(engine->mmio_base));
 185                 if (val == 0) /* powerwell off => engine idle */
 186                         continue;
 187
 188                 if (val & RING_WAIT)
 189                         add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns);
 190                 if (val & RING_WAIT_SEMAPHORE)
 191                         add_sample(&pmu->sample[I915_SAMPLE_SEMA], period_ns);
 192
 193                 /*
 194                  * While waiting on a semaphore or event, MI_MODE reports the
 195                  * ring as idle. However, previously using the seqno, and with
 196                  * execlists sampling, we account for the ring waiting as the
 197                  * engine being busy. Therefore, we record the sample as being
 198                  * busy if either waiting or !idle.
 199                  */
 200                 busy = val & (RING_WAIT_SEMAPHORE | RING_WAIT);
 201                 if (!busy) {
 202                         val = I915_READ_FW(RING_MI_MODE(engine->mmio_base));
 203                         busy = !(val & MODE_IDLE);
 204                 }
 205                 if (busy)
 206                         add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns);
 207         }
 208         spin_unlock_irqrestore(&dev_priv->uncore.lock, flags);
 209
 210         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
 211 }
 212
 213 static void
 214 add_sample_mult(struct i915_pmu_sample *sample, u32 val, u32 mul)
 215 {
 216         sample->cur += mul_u32_u32(val, mul);
 217 }
 218
 219 static void
 220 frequency_sample(struct drm_i915_private *dev_priv, unsigned int period_ns)
 221 {
 222         if (dev_priv->pmu.enable &
 223             config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) {
 224                 u32 val;
 225
 226                 val = dev_priv->gt_pm.rps.cur_freq;
 227                 if (dev_priv->gt.awake) {
 228                         intel_wakeref_t wakeref;
 229
 230                         with_intel_runtime_pm_if_in_use(&dev_priv->runtime_pm,
 231                                                         wakeref) {
 232                                 val = intel_uncore_read_notrace(&dev_priv->uncore,
 233                                                                 GEN6_RPSTAT1);
 234                                 val = intel_get_cagf(dev_priv, val);
 235                         }
 236                 }
 237
 238                 add_sample_mult(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_ACT],
 239                                 intel_gpu_freq(dev_priv, val),
 240                                 period_ns / 1000);
 241         }
 242
 243         if (dev_priv->pmu.enable &
 244             config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) {
 245                 add_sample_mult(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_REQ],
 246                                 intel_gpu_freq(dev_priv,
 247                                                dev_priv->gt_pm.rps.cur_freq),
 248                                 period_ns / 1000);
 249         }
 250 }
 251
 252 static enum hrtimer_restart i915_sample(struct hrtimer *hrtimer)
 253 {
 254         struct drm_i915_private *i915 =
 255                 container_of(hrtimer, struct drm_i915_private, pmu.timer);
 256         unsigned int period_ns;
 257         ktime_t now;
 258
 259         if (!READ_ONCE(i915->pmu.timer_enabled))
 260                 return HRTIMER_NORESTART;
 261
 262         now = ktime_get();
 263         period_ns = ktime_to_ns(ktime_sub(now, i915->pmu.timer_last));
 264         i915->pmu.timer_last = now;
 265
 266         /*
 267          * Strictly speaking the passed in period may not be 100% accurate for
 268          * all internal calculation, since some amount of time can be spent on
 269          * grabbing the forcewake. However the potential error from timer call-
 270          * back delay greatly dominates this so we keep it simple.
 271          */
 272         engines_sample(i915, period_ns);
 273         frequency_sample(i915, period_ns);
 274
 275         hrtimer_forward(hrtimer, now, ns_to_ktime(PERIOD));
 276
 277         return HRTIMER_RESTART;
 278 }
 279
 280 static u64 count_interrupts(struct drm_i915_private *i915)
 281 {
 282         /* open-coded kstat_irqs() */
 283         struct irq_desc *desc = irq_to_desc(i915->drm.pdev->irq);
 284         u64 sum = 0;
 285         int cpu;
 286
 287         if (!desc || !desc->kstat_irqs)
 288                 return 0;
 289
 290         for_each_possible_cpu(cpu)
 291                 sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
 292
 293         return sum;
 294 }
 295
 296 static void engine_event_destroy(struct perf_event *event)
 297 {
 298         struct drm_i915_private *i915 =
 299                 container_of(event->pmu, typeof(*i915), pmu.base);
 300         struct intel_engine_cs *engine;
 301
 302         engine = intel_engine_lookup_user(i915,
 303                                           engine_event_class(event),
 304                                           engine_event_instance(event));
 305         if (WARN_ON_ONCE(!engine))
 306                 return;
 307
 308         if (engine_event_sample(event) == I915_SAMPLE_BUSY &&
 309             intel_engine_supports_stats(engine))
 310                 intel_disable_engine_stats(engine);
 311 }
 312
 313 static void i915_pmu_event_destroy(struct perf_event *event)
 314 {
 315         WARN_ON(event->parent);
 316
 317         if (is_engine_event(event))
 318                 engine_event_destroy(event);
 319 }
 320
 321 static int
 322 engine_event_status(struct intel_engine_cs *engine,
 323                     enum drm_i915_pmu_engine_sample sample)
 324 {
 325         switch (sample) {
 326         case I915_SAMPLE_BUSY:
 327         case I915_SAMPLE_WAIT:
 328                 break;
 329         case I915_SAMPLE_SEMA:
 330                 if (INTEL_GEN(engine->i915) < 6)
 331                         return -ENODEV;
 332                 break;
 333         default:
 334                 return -ENOENT;
 335         }
 336
 337         return 0;
 338 }
 339
 340 static int
 341 config_status(struct drm_i915_private *i915, u64 config)
 342 {
 343         switch (config) {
 344         case I915_PMU_ACTUAL_FREQUENCY:
 345                 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
 346                         /* Requires a mutex for sampling! */
 347                         return -ENODEV;
 348                 /* Fall-through. */
 349         case I915_PMU_REQUESTED_FREQUENCY:
 350                 if (INTEL_GEN(i915) < 6)
 351                         return -ENODEV;
 352                 break;
 353         case I915_PMU_INTERRUPTS:
 354                 break;
 355         case I915_PMU_RC6_RESIDENCY:
 356                 if (!HAS_RC6(i915))
 357                         return -ENODEV;
 358                 break;
 359         default:
 360                 return -ENOENT;
 361         }
 362
 363         return 0;
 364 }
 365
 366 static int engine_event_init(struct perf_event *event)
 367 {
 368         struct drm_i915_private *i915 =
 369                 container_of(event->pmu, typeof(*i915), pmu.base);
 370         struct intel_engine_cs *engine;
 371         u8 sample;
 372         int ret;
 373
 374         engine = intel_engine_lookup_user(i915, engine_event_class(event),
 375                                           engine_event_instance(event));
 376         if (!engine)
 377                 return -ENODEV;
 378
 379         sample = engine_event_sample(event);
 380         ret = engine_event_status(engine, sample);
 381         if (ret)
 382                 return ret;
 383
 384         if (sample == I915_SAMPLE_BUSY && intel_engine_supports_stats(engine))
 385                 ret = intel_enable_engine_stats(engine);
 386
 387         return ret;
 388 }
 389
 390 static int i915_pmu_event_init(struct perf_event *event)
 391 {
 392         struct drm_i915_private *i915 =
 393                 container_of(event->pmu, typeof(*i915), pmu.base);
 394         int ret;
 395
 396         if (event->attr.type != event->pmu->type)
 397                 return -ENOENT;
 398
 399         /* unsupported modes and filters */
 400         if (event->attr.sample_period) /* no sampling */
 401                 return -EINVAL;
 402
 403         if (has_branch_stack(event))
 404                 return -EOPNOTSUPP;
 405
 406         if (event->cpu < 0)
 407                 return -EINVAL;
 408
 409         /* only allow running on one cpu at a time */
 410         if (!cpumask_test_cpu(event->cpu, &i915_pmu_cpumask))
 411                 return -EINVAL;
 412
 413         if (is_engine_event(event))
 414                 ret = engine_event_init(event);
 415         else
 416                 ret = config_status(i915, event->attr.config);
 417         if (ret)
 418                 return ret;
 419
 420         if (!event->parent)
 421                 event->destroy = i915_pmu_event_destroy;
 422
 423         return 0;
 424 }
 425
 426 static u64 __get_rc6(struct drm_i915_private *i915)
 427 {
 428         u64 val;
 429
 430         val = intel_rc6_residency_ns(i915,
 431                                      IS_VALLEYVIEW(i915) ?
 432                                      VLV_GT_RENDER_RC6 :
 433                                      GEN6_GT_GFX_RC6);
 434
 435         if (HAS_RC6p(i915))
 436                 val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6p);
 437
 438         if (HAS_RC6pp(i915))
 439                 val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6pp);
 440
 441         return val;
 442 }
 443
 444 static u64 get_rc6(struct drm_i915_private *i915)
 445 {
 446 #if IS_ENABLED(CONFIG_PM)
 447         struct intel_runtime_pm *rpm = &i915->runtime_pm;
 448         intel_wakeref_t wakeref;
 449         unsigned long flags;
 450         u64 val;
 451
 452         wakeref = intel_runtime_pm_get_if_in_use(rpm);
 453         if (wakeref) {
 454                 val = __get_rc6(i915);
 455                 intel_runtime_pm_put(rpm, wakeref);
 456
 457                 /*
 458                  * If we are coming back from being runtime suspended we must
 459                  * be careful not to report a larger value than returned
 460                  * previously.
 461                  */
 462
 463                 spin_lock_irqsave(&i915->pmu.lock, flags);
 464
 465                 if (val >= i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur) {
 466                         i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur = 0;
 467                         i915->pmu.sample[__I915_SAMPLE_RC6].cur = val;
 468                 } else {
 469                         val = i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur;
 470                 }
 471
 472                 spin_unlock_irqrestore(&i915->pmu.lock, flags);
 473         } else {
 474                 struct device *kdev = rpm->kdev;
 475
 476                 /*
 477                  * We are runtime suspended.
 478                  *
 479                  * Report the delta from when the device was suspended to now,
 480                  * on top of the last known real value, as the approximated RC6
 481                  * counter value.
 482                  */
 483                 spin_lock_irqsave(&i915->pmu.lock, flags);
 484
 485                 /*
 486                  * After the above branch intel_runtime_pm_get_if_in_use failed
 487                  * to get the runtime PM reference we cannot assume we are in
 488                  * runtime suspend since we can either: a) race with coming out
 489                  * of it before we took the power.lock, or b) there are other
 490                  * states than suspended which can bring us here.
 491                  *
 492                  * We need to double-check that we are indeed currently runtime
 493                  * suspended and if not we cannot do better than report the last
 494                  * known RC6 value.
 495                  */
 496                 if (pm_runtime_status_suspended(kdev)) {
 497                         val = pm_runtime_suspended_time(kdev);
 498
 499                         if (!i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur)
 500                                 i915->pmu.suspended_time_last = val;
 501
 502                         val -= i915->pmu.suspended_time_last;
 503                         val += i915->pmu.sample[__I915_SAMPLE_RC6].cur;
 504
 505                         i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur = val;
 506                 } else if (i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur) {
 507                         val = i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur;
 508                 } else {
 509                         val = i915->pmu.sample[__I915_SAMPLE_RC6].cur;
 510                 }
 511
 512                 spin_unlock_irqrestore(&i915->pmu.lock, flags);
 513         }
 514
 515         return val;
 516 #else
 517         return __get_rc6(i915);
 518 #endif
 519 }
 520
 521 static u64 __i915_pmu_event_read(struct perf_event *event)
 522 {
 523         struct drm_i915_private *i915 =
 524                 container_of(event->pmu, typeof(*i915), pmu.base);
 525         u64 val = 0;
 526
 527         if (is_engine_event(event)) {
 528                 u8 sample = engine_event_sample(event);
 529                 struct intel_engine_cs *engine;
 530
 531                 engine = intel_engine_lookup_user(i915,
 532                                                   engine_event_class(event),
 533                                                   engine_event_instance(event));
 534
 535                 if (WARN_ON_ONCE(!engine)) {
 536                         /* Do nothing */
 537                 } else if (sample == I915_SAMPLE_BUSY &&
 538                            intel_engine_supports_stats(engine)) {
 539                         val = ktime_to_ns(intel_engine_get_busy_time(engine));
 540                 } else {
 541                         val = engine->pmu.sample[sample].cur;
 542                 }
 543         } else {
 544                 switch (event->attr.config) {
 545                 case I915_PMU_ACTUAL_FREQUENCY:
 546                         val =
 547                            div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_ACT].cur,
 548                                    USEC_PER_SEC /* to MHz */);
 549                         break;
 550                 case I915_PMU_REQUESTED_FREQUENCY:
 551                         val =
 552                            div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_REQ].cur,
 553                                    USEC_PER_SEC /* to MHz */);
 554                         break;
 555                 case I915_PMU_INTERRUPTS:
 556                         val = count_interrupts(i915);
 557                         break;
 558                 case I915_PMU_RC6_RESIDENCY:
 559                         val = get_rc6(i915);
 560                         break;
 561                 }
 562         }
 563
 564         return val;
 565 }
 566
 567 static void i915_pmu_event_read(struct perf_event *event)
 568 {
 569         struct hw_perf_event *hwc = &event->hw;
 570         u64 prev, new;
 571
 572 again:
 573         prev = local64_read(&hwc->prev_count);
 574         new = __i915_pmu_event_read(event);
 575
 576         if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev)
 577                 goto again;
 578
 579         local64_add(new - prev, &event->count);
 580 }
 581
 582 static void i915_pmu_enable(struct perf_event *event)
 583 {
 584         struct drm_i915_private *i915 =
 585                 container_of(event->pmu, typeof(*i915), pmu.base);
 586         unsigned int bit = event_enabled_bit(event);
 587         unsigned long flags;
 588
 589         spin_lock_irqsave(&i915->pmu.lock, flags);
 590
 591         /*
 592          * Update the bitmask of enabled events and increment
 593          * the event reference counter.
 594          */
 595         BUILD_BUG_ON(ARRAY_SIZE(i915->pmu.enable_count) != I915_PMU_MASK_BITS);
 596         GEM_BUG_ON(bit >= ARRAY_SIZE(i915->pmu.enable_count));
 597         GEM_BUG_ON(i915->pmu.enable_count[bit] == ~0);
 598         i915->pmu.enable |= BIT_ULL(bit);
 599         i915->pmu.enable_count[bit]++;
 600
 601         /*
 602          * Start the sampling timer if needed and not already enabled.
 603          */
 604         __i915_pmu_maybe_start_timer(i915);
 605
 606         /*
 607          * For per-engine events the bitmask and reference counting
 608          * is stored per engine.
 609          */
 610         if (is_engine_event(event)) {
 611                 u8 sample = engine_event_sample(event);
 612                 struct intel_engine_cs *engine;
 613
 614                 engine = intel_engine_lookup_user(i915,
 615                                                   engine_event_class(event),
 616                                                   engine_event_instance(event));
 617
 618                 BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.enable_count) !=
 619                              I915_ENGINE_SAMPLE_COUNT);
 620                 BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.sample) !=
 621                              I915_ENGINE_SAMPLE_COUNT);
 622                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count));
 623                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample));
 624                 GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0);
 625
 626                 engine->pmu.enable |= BIT(sample);
 627                 engine->pmu.enable_count[sample]++;
 628         }
 629
 630         spin_unlock_irqrestore(&i915->pmu.lock, flags);
 631
 632         /*
 633          * Store the current counter value so we can report the correct delta
 634          * for all listeners. Even when the event was already enabled and has
 635          * an existing non-zero value.
 636          */
 637         local64_set(&event->hw.prev_count, __i915_pmu_event_read(event));
 638 }
 639
 640 static void i915_pmu_disable(struct perf_event *event)
 641 {
 642         struct drm_i915_private *i915 =
 643                 container_of(event->pmu, typeof(*i915), pmu.base);
 644         unsigned int bit = event_enabled_bit(event);
 645         unsigned long flags;
 646
 647         spin_lock_irqsave(&i915->pmu.lock, flags);
 648
 649         if (is_engine_event(event)) {
 650                 u8 sample = engine_event_sample(event);
 651                 struct intel_engine_cs *engine;
 652
 653                 engine = intel_engine_lookup_user(i915,
 654                                                   engine_event_class(event),
 655                                                   engine_event_instance(event));
 656
 657                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count));
 658                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample));
 659                 GEM_BUG_ON(engine->pmu.enable_count[sample] == 0);
 660
 661                 /*
 662                  * Decrement the reference count and clear the enabled
 663                  * bitmask when the last listener on an event goes away.
 664                  */
 665                 if (--engine->pmu.enable_count[sample] == 0)
 666                         engine->pmu.enable &= ~BIT(sample);
 667         }
 668
 669         GEM_BUG_ON(bit >= ARRAY_SIZE(i915->pmu.enable_count));
 670         GEM_BUG_ON(i915->pmu.enable_count[bit] == 0);
 671         /*
 672          * Decrement the reference count and clear the enabled
 673          * bitmask when the last listener on an event goes away.
 674          */
 675         if (--i915->pmu.enable_count[bit] == 0) {
 676                 i915->pmu.enable &= ~BIT_ULL(bit);
 677                 i915->pmu.timer_enabled &= pmu_needs_timer(i915, true);
 678         }
 679
 680         spin_unlock_irqrestore(&i915->pmu.lock, flags);
 681 }
 682
 683 static void i915_pmu_event_start(struct perf_event *event, int flags)
 684 {
 685         i915_pmu_enable(event);
 686         event->hw.state = 0;
 687 }
 688
 689 static void i915_pmu_event_stop(struct perf_event *event, int flags)
 690 {
 691         if (flags & PERF_EF_UPDATE)
 692                 i915_pmu_event_read(event);
 693         i915_pmu_disable(event);
 694         event->hw.state = PERF_HES_STOPPED;
 695 }
 696
 697 static int i915_pmu_event_add(struct perf_event *event, int flags)
 698 {
 699         if (flags & PERF_EF_START)
 700                 i915_pmu_event_start(event, flags);
 701
 702         return 0;
 703 }
 704
 705 static void i915_pmu_event_del(struct perf_event *event, int flags)
 706 {
 707         i915_pmu_event_stop(event, PERF_EF_UPDATE);
 708 }
 709
 710 static int i915_pmu_event_event_idx(struct perf_event *event)
 711 {
 712         return 0;
 713 }
 714
 715 struct i915_str_attribute {
 716         struct device_attribute attr;
 717         const char *str;
 718 };
 719
 720 static ssize_t i915_pmu_format_show(struct device *dev,
 721                                     struct device_attribute *attr, char *buf)
 722 {
 723         struct i915_str_attribute *eattr;
 724
 725         eattr = container_of(attr, struct i915_str_attribute, attr);
 726         return sprintf(buf, "%s\n", eattr->str);
 727 }
 728
 729 #define I915_PMU_FORMAT_ATTR(_name, _config) \
 730         (&((struct i915_str_attribute[]) { \
 731                 { .attr = __ATTR(_name, 0444, i915_pmu_format_show, NULL), \
 732                   .str = _config, } \
 733         })[0].attr.attr)
 734
 735 static struct attribute *i915_pmu_format_attrs[] = {
 736         I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"),
 737         NULL,
 738 };
 739
 740 static const struct attribute_group i915_pmu_format_attr_group = {
 741         .name = "format",
 742         .attrs = i915_pmu_format_attrs,
 743 };
 744
 745 struct i915_ext_attribute {
 746         struct device_attribute attr;
 747         unsigned long val;
 748 };
 749
 750 static ssize_t i915_pmu_event_show(struct device *dev,
 751                                    struct device_attribute *attr, char *buf)
 752 {
 753         struct i915_ext_attribute *eattr;
 754
 755         eattr = container_of(attr, struct i915_ext_attribute, attr);
 756         return sprintf(buf, "config=0x%lx\n", eattr->val);
 757 }
 758
 759 static struct attribute_group i915_pmu_events_attr_group = {
 760         .name = "events",
 761         /* Patch in attrs at runtime. */
 762 };
 763
 764 static ssize_t
 765 i915_pmu_get_attr_cpumask(struct device *dev,
 766                           struct device_attribute *attr,
 767                           char *buf)
 768 {
 769         return cpumap_print_to_pagebuf(true, buf, &i915_pmu_cpumask);
 770 }
 771
 772 static DEVICE_ATTR(cpumask, 0444, i915_pmu_get_attr_cpumask, NULL);
 773
 774 static struct attribute *i915_cpumask_attrs[] = {
 775         &dev_attr_cpumask.attr,
 776         NULL,
 777 };
 778
 779 static const struct attribute_group i915_pmu_cpumask_attr_group = {
 780         .attrs = i915_cpumask_attrs,
 781 };
 782
 783 static const struct attribute_group *i915_pmu_attr_groups[] = {
 784         &i915_pmu_format_attr_group,
 785         &i915_pmu_events_attr_group,
 786         &i915_pmu_cpumask_attr_group,
 787         NULL
 788 };
 789
 790 #define __event(__config, __name, __unit) \
 791 { \
 792         .config = (__config), \
 793         .name = (__name), \
 794         .unit = (__unit), \
 795 }
 796
 797 #define __engine_event(__sample, __name) \
 798 { \
 799         .sample = (__sample), \
 800         .name = (__name), \
 801 }
 802
 803 static struct i915_ext_attribute *
 804 add_i915_attr(struct i915_ext_attribute *attr, const char *name, u64 config)
 805 {
 806         sysfs_attr_init(&attr->attr.attr);
 807         attr->attr.attr.name = name;
 808         attr->attr.attr.mode = 0444;
 809         attr->attr.show = i915_pmu_event_show;
 810         attr->val = config;
 811
 812         return ++attr;
 813 }
 814
 815 static struct perf_pmu_events_attr *
 816 add_pmu_attr(struct perf_pmu_events_attr *attr, const char *name,
 817              const char *str)
 818 {
 819         sysfs_attr_init(&attr->attr.attr);
 820         attr->attr.attr.name = name;
 821         attr->attr.attr.mode = 0444;
 822         attr->attr.show = perf_event_sysfs_show;
 823         attr->event_str = str;
 824
 825         return ++attr;
 826 }
 827
 828 static struct attribute **
 829 create_event_attributes(struct drm_i915_private *i915)
 830 {
 831         static const struct {
 832                 u64 config;
 833                 const char *name;
 834                 const char *unit;
 835         } events[] = {
 836                 __event(I915_PMU_ACTUAL_FREQUENCY, "actual-frequency", "MHz"),
 837                 __event(I915_PMU_REQUESTED_FREQUENCY, "requested-frequency", "MHz"),
 838                 __event(I915_PMU_INTERRUPTS, "interrupts", NULL),
 839                 __event(I915_PMU_RC6_RESIDENCY, "rc6-residency", "ns"),
 840         };
 841         static const struct {
 842                 enum drm_i915_pmu_engine_sample sample;
 843                 char *name;
 844         } engine_events[] = {
 845                 __engine_event(I915_SAMPLE_BUSY, "busy"),
 846                 __engine_event(I915_SAMPLE_SEMA, "sema"),
 847                 __engine_event(I915_SAMPLE_WAIT, "wait"),
 848         };
 849         unsigned int count = 0;
 850         struct perf_pmu_events_attr *pmu_attr = NULL, *pmu_iter;
 851         struct i915_ext_attribute *i915_attr = NULL, *i915_iter;
 852         struct attribute **attr = NULL, **attr_iter;
 853         struct intel_engine_cs *engine;
 854         enum intel_engine_id id;
 855         unsigned int i;
 856
 857         /* Count how many counters we will be exposing. */
 858         for (i = 0; i < ARRAY_SIZE(events); i++) {
 859                 if (!config_status(i915, events[i].config))
 860                         count++;
 861         }
 862
 863         for_each_engine(engine, i915, id) {
 864                 for (i = 0; i < ARRAY_SIZE(engine_events); i++) {
 865                         if (!engine_event_status(engine,
 866                                                  engine_events[i].sample))
 867                                 count++;
 868                 }
 869         }
 870
 871         /* Allocate attribute objects and table. */
 872         i915_attr = kcalloc(count, sizeof(*i915_attr), GFP_KERNEL);
 873         if (!i915_attr)
 874                 goto err_alloc;
 875
 876         pmu_attr = kcalloc(count, sizeof(*pmu_attr), GFP_KERNEL);
 877         if (!pmu_attr)
 878                 goto err_alloc;
 879
 880         /* Max one pointer of each attribute type plus a termination entry. */
 881         attr = kcalloc(count * 2 + 1, sizeof(*attr), GFP_KERNEL);
 882         if (!attr)
 883                 goto err_alloc;
 884
 885         i915_iter = i915_attr;
 886         pmu_iter = pmu_attr;
 887         attr_iter = attr;
 888
 889         /* Initialize supported non-engine counters. */
 890         for (i = 0; i < ARRAY_SIZE(events); i++) {
 891                 char *str;
 892
 893                 if (config_status(i915, events[i].config))
 894                         continue;
 895
 896                 str = kstrdup(events[i].name, GFP_KERNEL);
 897                 if (!str)
 898                         goto err;
 899
 900                 *attr_iter++ = &i915_iter->attr.attr;
 901                 i915_iter = add_i915_attr(i915_iter, str, events[i].config);
 902
 903                 if (events[i].unit) {
 904                         str = kasprintf(GFP_KERNEL, "%s.unit", events[i].name);
 905                         if (!str)
 906                                 goto err;
 907
 908                         *attr_iter++ = &pmu_iter->attr.attr;
 909                         pmu_iter = add_pmu_attr(pmu_iter, str, events[i].unit);
 910                 }
 911         }
 912
 913         /* Initialize supported engine counters. */
 914         for_each_engine(engine, i915, id) {
 915                 for (i = 0; i < ARRAY_SIZE(engine_events); i++) {
 916                         char *str;
 917
 918                         if (engine_event_status(engine,
 919                                                 engine_events[i].sample))
 920                                 continue;
 921
 922                         str = kasprintf(GFP_KERNEL, "%s-%s",
 923                                         engine->name, engine_events[i].name);
 924                         if (!str)
 925                                 goto err;
 926
 927                         *attr_iter++ = &i915_iter->attr.attr;
 928                         i915_iter =
 929                                 add_i915_attr(i915_iter, str,
 930                                               __I915_PMU_ENGINE(engine->uabi_class,
 931                                                                 engine->instance,
 932                                                                 engine_events[i].sample));
 933
 934                         str = kasprintf(GFP_KERNEL, "%s-%s.unit",
 935                                         engine->name, engine_events[i].name);
 936                         if (!str)
 937                                 goto err;
 938
 939                         *attr_iter++ = &pmu_iter->attr.attr;
 940                         pmu_iter = add_pmu_attr(pmu_iter, str, "ns");
 941                 }
 942         }
 943
 944         i915->pmu.i915_attr = i915_attr;
 945         i915->pmu.pmu_attr = pmu_attr;
 946
 947         return attr;
 948
 949 err:;
 950         for (attr_iter = attr; *attr_iter; attr_iter++)
 951                 kfree((*attr_iter)->name);
 952
 953 err_alloc:
 954         kfree(attr);
 955         kfree(i915_attr);
 956         kfree(pmu_attr);
 957
 958         return NULL;
 959 }
 960
 961 static void free_event_attributes(struct drm_i915_private *i915)
 962 {
 963         struct attribute **attr_iter = i915_pmu_events_attr_group.attrs;
 964
 965         for (; *attr_iter; attr_iter++)
 966                 kfree((*attr_iter)->name);
 967
 968         kfree(i915_pmu_events_attr_group.attrs);
 969         kfree(i915->pmu.i915_attr);
 970         kfree(i915->pmu.pmu_attr);
 971
 972         i915_pmu_events_attr_group.attrs = NULL;
 973         i915->pmu.i915_attr = NULL;
 974         i915->pmu.pmu_attr = NULL;
 975 }
 976
 977 static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
 978 {
 979         struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node);
 980
 981         GEM_BUG_ON(!pmu->base.event_init);
 982
 983         /* Select the first online CPU as a designated reader. */
 984         if (!cpumask_weight(&i915_pmu_cpumask))
 985                 cpumask_set_cpu(cpu, &i915_pmu_cpumask);
 986
 987         return 0;
 988 }
 989
 990 static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node)
 991 {
 992         struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node);
 993         unsigned int target;
 994
 995         GEM_BUG_ON(!pmu->base.event_init);
 996
 997         if (cpumask_test_and_clear_cpu(cpu, &i915_pmu_cpumask)) {
 998                 target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu);
 999                 /* Migrate events if there is a valid target */
1000                 if (target < nr_cpu_ids) {
1001                         cpumask_set_cpu(target, &i915_pmu_cpumask);
1002                         perf_pmu_migrate_context(&pmu->base, cpu, target);
1003                 }
1004         }
1005
1006         return 0;
1007 }
1008
1009 static enum cpuhp_state cpuhp_slot = CPUHP_INVALID;
1010
1011 static int i915_pmu_register_cpuhp_state(struct drm_i915_private *i915)
1012 {
1013         enum cpuhp_state slot;
1014         int ret;
1015
1016         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
1017                                       "perf/x86/intel/i915:online",
1018                                       i915_pmu_cpu_online,
1019                                       i915_pmu_cpu_offline);
1020         if (ret < 0)
1021                 return ret;
1022
1023         slot = ret;
1024         ret = cpuhp_state_add_instance(slot, &i915->pmu.node);
1025         if (ret) {
1026                 cpuhp_remove_multi_state(slot);
1027                 return ret;
1028         }
1029
1030         cpuhp_slot = slot;
1031         return 0;
1032 }
1033
1034 static void i915_pmu_unregister_cpuhp_state(struct drm_i915_private *i915)
1035 {
1036         WARN_ON(cpuhp_slot == CPUHP_INVALID);
1037         WARN_ON(cpuhp_state_remove_instance(cpuhp_slot, &i915->pmu.node));
1038         cpuhp_remove_multi_state(cpuhp_slot);
1039 }
1040
1041 void i915_pmu_register(struct drm_i915_private *i915)
1042 {
1043         int ret;
1044
1045         if (INTEL_GEN(i915) <= 2) {
1046                 DRM_INFO("PMU not supported for this GPU.");
1047                 return;
1048         }
1049
1050         i915_pmu_events_attr_group.attrs = create_event_attributes(i915);
1051         if (!i915_pmu_events_attr_group.attrs) {
1052                 ret = -ENOMEM;
1053                 goto err;
1054         }
1055
1056         i915->pmu.base.attr_groups      = i915_pmu_attr_groups;
1057         i915->pmu.base.task_ctx_nr      = perf_invalid_context;
1058         i915->pmu.base.event_init       = i915_pmu_event_init;
1059         i915->pmu.base.add              = i915_pmu_event_add;
1060         i915->pmu.base.del              = i915_pmu_event_del;
1061         i915->pmu.base.start            = i915_pmu_event_start;
1062         i915->pmu.base.stop             = i915_pmu_event_stop;
1063         i915->pmu.base.read             = i915_pmu_event_read;
1064         i915->pmu.base.event_idx        = i915_pmu_event_event_idx;
1065
1066         spin_lock_init(&i915->pmu.lock);
1067         hrtimer_init(&i915->pmu.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1068         i915->pmu.timer.function = i915_sample;
1069
1070         ret = perf_pmu_register(&i915->pmu.base, "i915", -1);
1071         if (ret)
1072                 goto err;
1073
1074         ret = i915_pmu_register_cpuhp_state(i915);
1075         if (ret)
1076                 goto err_unreg;
1077
1078         return;
1079
1080 err_unreg:
1081         perf_pmu_unregister(&i915->pmu.base);
1082 err:
1083         i915->pmu.base.event_init = NULL;
1084         free_event_attributes(i915);
1085         DRM_NOTE("Failed to register PMU! (err=%d)\n", ret);
1086 }
1087
1088 void i915_pmu_unregister(struct drm_i915_private *i915)
1089 {
1090         if (!i915->pmu.base.event_init)
1091                 return;
1092
1093         WARN_ON(i915->pmu.enable);
1094
1095         hrtimer_cancel(&i915->pmu.timer);
1096
1097         i915_pmu_unregister_cpuhp_state(i915);
1098
1099         perf_pmu_unregister(&i915->pmu.base);
1100         i915->pmu.base.event_init = NULL;
1101         free_event_attributes(i915);
1102 }