drivers/hwtracing/coresight/coresight-etm4x.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2014, The Linux Foundation. All rights reserved.
   4  */
   5
   6 #include <linux/kernel.h>
   7 #include <linux/moduleparam.h>
   8 #include <linux/init.h>
   9 #include <linux/types.h>
  10 #include <linux/device.h>
  11 #include <linux/io.h>
  12 #include <linux/err.h>
  13 #include <linux/fs.h>
  14 #include <linux/slab.h>
  15 #include <linux/delay.h>
  16 #include <linux/smp.h>
  17 #include <linux/sysfs.h>
  18 #include <linux/stat.h>
  19 #include <linux/clk.h>
  20 #include <linux/cpu.h>
  21 #include <linux/coresight.h>
  22 #include <linux/coresight-pmu.h>
  23 #include <linux/pm_wakeup.h>
  24 #include <linux/amba/bus.h>
  25 #include <linux/seq_file.h>
  26 #include <linux/uaccess.h>
  27 #include <linux/perf_event.h>
  28 #include <linux/pm_runtime.h>
  29 #include <asm/sections.h>
  30 #include <asm/local.h>
  31 #include <asm/virt.h>
  32
  33 #include "coresight-etm4x.h"
  34 #include "coresight-etm-perf.h"
  35
  36 static int boot_enable;
  37 module_param(boot_enable, int, 0444);
  38 MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
  39
  40 /* The number of ETMv4 currently registered */
  41 static int etm4_count;
  42 static struct etmv4_drvdata *etmdrvdata[NR_CPUS];
  43 static void etm4_set_default_config(struct etmv4_config *config);
  44 static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
  45                                   struct perf_event *event);
  46
  47 static enum cpuhp_state hp_online;
  48
  49 static void etm4_os_unlock(struct etmv4_drvdata *drvdata)
  50 {
  51         /* Writing 0 to TRCOSLAR unlocks the trace registers */
  52         writel_relaxed(0x0, drvdata->base + TRCOSLAR);
  53         drvdata->os_unlock = true;
  54         isb();
  55 }
  56
  57 static bool etm4_arch_supported(u8 arch)
  58 {
  59         /* Mask out the minor version number */
  60         switch (arch & 0xf0) {
  61         case ETM_ARCH_V4:
  62                 break;
  63         default:
  64                 return false;
  65         }
  66         return true;
  67 }
  68
  69 static int etm4_cpu_id(struct coresight_device *csdev)
  70 {
  71         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
  72
  73         return drvdata->cpu;
  74 }
  75
  76 static int etm4_trace_id(struct coresight_device *csdev)
  77 {
  78         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
  79
  80         return drvdata->trcid;
  81 }
  82
  83 struct etm4_enable_arg {
  84         struct etmv4_drvdata *drvdata;
  85         int rc;
  86 };
  87
  88 static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
  89 {
  90         int i, rc;
  91         struct etmv4_config *config = &drvdata->config;
  92         struct device *etm_dev = &drvdata->csdev->dev;
  93
  94         CS_UNLOCK(drvdata->base);
  95
  96         etm4_os_unlock(drvdata);
  97
  98         rc = coresight_claim_device_unlocked(drvdata->base);
  99         if (rc)
 100                 goto done;
 101
 102         /* Disable the trace unit before programming trace registers */
 103         writel_relaxed(0, drvdata->base + TRCPRGCTLR);
 104
 105         /* wait for TRCSTATR.IDLE to go up */
 106         if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 1))
 107                 dev_err(etm_dev,
 108                         "timeout while waiting for Idle Trace Status\n");
 109
 110         writel_relaxed(config->pe_sel, drvdata->base + TRCPROCSELR);
 111         writel_relaxed(config->cfg, drvdata->base + TRCCONFIGR);
 112         /* nothing specific implemented */
 113         writel_relaxed(0x0, drvdata->base + TRCAUXCTLR);
 114         writel_relaxed(config->eventctrl0, drvdata->base + TRCEVENTCTL0R);
 115         writel_relaxed(config->eventctrl1, drvdata->base + TRCEVENTCTL1R);
 116         writel_relaxed(config->stall_ctrl, drvdata->base + TRCSTALLCTLR);
 117         writel_relaxed(config->ts_ctrl, drvdata->base + TRCTSCTLR);
 118         writel_relaxed(config->syncfreq, drvdata->base + TRCSYNCPR);
 119         writel_relaxed(config->ccctlr, drvdata->base + TRCCCCTLR);
 120         writel_relaxed(config->bb_ctrl, drvdata->base + TRCBBCTLR);
 121         writel_relaxed(drvdata->trcid, drvdata->base + TRCTRACEIDR);
 122         writel_relaxed(config->vinst_ctrl, drvdata->base + TRCVICTLR);
 123         writel_relaxed(config->viiectlr, drvdata->base + TRCVIIECTLR);
 124         writel_relaxed(config->vissctlr,
 125                        drvdata->base + TRCVISSCTLR);
 126         writel_relaxed(config->vipcssctlr,
 127                        drvdata->base + TRCVIPCSSCTLR);
 128         for (i = 0; i < drvdata->nrseqstate - 1; i++)
 129                 writel_relaxed(config->seq_ctrl[i],
 130                                drvdata->base + TRCSEQEVRn(i));
 131         writel_relaxed(config->seq_rst, drvdata->base + TRCSEQRSTEVR);
 132         writel_relaxed(config->seq_state, drvdata->base + TRCSEQSTR);
 133         writel_relaxed(config->ext_inp, drvdata->base + TRCEXTINSELR);
 134         for (i = 0; i < drvdata->nr_cntr; i++) {
 135                 writel_relaxed(config->cntrldvr[i],
 136                                drvdata->base + TRCCNTRLDVRn(i));
 137                 writel_relaxed(config->cntr_ctrl[i],
 138                                drvdata->base + TRCCNTCTLRn(i));
 139                 writel_relaxed(config->cntr_val[i],
 140                                drvdata->base + TRCCNTVRn(i));
 141         }
 142
 143         /*
 144          * Resource selector pair 0 is always implemented and reserved.  As
 145          * such start at 2.
 146          */
 147         for (i = 2; i < drvdata->nr_resource * 2; i++)
 148                 writel_relaxed(config->res_ctrl[i],
 149                                drvdata->base + TRCRSCTLRn(i));
 150
 151         for (i = 0; i < drvdata->nr_ss_cmp; i++) {
 152                 writel_relaxed(config->ss_ctrl[i],
 153                                drvdata->base + TRCSSCCRn(i));
 154                 writel_relaxed(config->ss_status[i],
 155                                drvdata->base + TRCSSCSRn(i));
 156                 writel_relaxed(config->ss_pe_cmp[i],
 157                                drvdata->base + TRCSSPCICRn(i));
 158         }
 159         for (i = 0; i < drvdata->nr_addr_cmp; i++) {
 160                 writeq_relaxed(config->addr_val[i],
 161                                drvdata->base + TRCACVRn(i));
 162                 writeq_relaxed(config->addr_acc[i],
 163                                drvdata->base + TRCACATRn(i));
 164         }
 165         for (i = 0; i < drvdata->numcidc; i++)
 166                 writeq_relaxed(config->ctxid_pid[i],
 167                                drvdata->base + TRCCIDCVRn(i));
 168         writel_relaxed(config->ctxid_mask0, drvdata->base + TRCCIDCCTLR0);
 169         writel_relaxed(config->ctxid_mask1, drvdata->base + TRCCIDCCTLR1);
 170
 171         for (i = 0; i < drvdata->numvmidc; i++)
 172                 writeq_relaxed(config->vmid_val[i],
 173                                drvdata->base + TRCVMIDCVRn(i));
 174         writel_relaxed(config->vmid_mask0, drvdata->base + TRCVMIDCCTLR0);
 175         writel_relaxed(config->vmid_mask1, drvdata->base + TRCVMIDCCTLR1);
 176
 177         /*
 178          * Request to keep the trace unit powered and also
 179          * emulation of powerdown
 180          */
 181         writel_relaxed(readl_relaxed(drvdata->base + TRCPDCR) | TRCPDCR_PU,
 182                        drvdata->base + TRCPDCR);
 183
 184         /* Enable the trace unit */
 185         writel_relaxed(1, drvdata->base + TRCPRGCTLR);
 186
 187         /* wait for TRCSTATR.IDLE to go back down to '0' */
 188         if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 0))
 189                 dev_err(etm_dev,
 190                         "timeout while waiting for Idle Trace Status\n");
 191
 192         /*
 193          * As recommended by section 4.3.7 ("Synchronization when using the
 194          * memory-mapped interface") of ARM IHI 0064D
 195          */
 196         dsb(sy);
 197         isb();
 198
 199 done:
 200         CS_LOCK(drvdata->base);
 201
 202         dev_dbg(etm_dev, "cpu: %d enable smp call done: %d\n",
 203                 drvdata->cpu, rc);
 204         return rc;
 205 }
 206
 207 static void etm4_enable_hw_smp_call(void *info)
 208 {
 209         struct etm4_enable_arg *arg = info;
 210
 211         if (WARN_ON(!arg))
 212                 return;
 213         arg->rc = etm4_enable_hw(arg->drvdata);
 214 }
 215
 216 /*
 217  * The goal of function etm4_config_timestamp_event() is to configure a
 218  * counter that will tell the tracer to emit a timestamp packet when it
 219  * reaches zero.  This is done in order to get a more fine grained idea
 220  * of when instructions are executed so that they can be correlated
 221  * with execution on other CPUs.
 222  *
 223  * To do this the counter itself is configured to self reload and
 224  * TRCRSCTLR1 (always true) used to get the counter to decrement.  From
 225  * there a resource selector is configured with the counter and the
 226  * timestamp control register to use the resource selector to trigger the
 227  * event that will insert a timestamp packet in the stream.
 228  */
 229 static int etm4_config_timestamp_event(struct etmv4_drvdata *drvdata)
 230 {
 231         int ctridx, ret = -EINVAL;
 232         int counter, rselector;
 233         u32 val = 0;
 234         struct etmv4_config *config = &drvdata->config;
 235
 236         /* No point in trying if we don't have at least one counter */
 237         if (!drvdata->nr_cntr)
 238                 goto out;
 239
 240         /* Find a counter that hasn't been initialised */
 241         for (ctridx = 0; ctridx < drvdata->nr_cntr; ctridx++)
 242                 if (config->cntr_val[ctridx] == 0)
 243                         break;
 244
 245         /* All the counters have been configured already, bail out */
 246         if (ctridx == drvdata->nr_cntr) {
 247                 pr_debug("%s: no available counter found\n", __func__);
 248                 ret = -ENOSPC;
 249                 goto out;
 250         }
 251
 252         /*
 253          * Searching for an available resource selector to use, starting at
 254          * '2' since every implementation has at least 2 resource selector.
 255          * ETMIDR4 gives the number of resource selector _pairs_,
 256          * hence multiply by 2.
 257          */
 258         for (rselector = 2; rselector < drvdata->nr_resource * 2; rselector++)
 259                 if (!config->res_ctrl[rselector])
 260                         break;
 261
 262         if (rselector == drvdata->nr_resource * 2) {
 263                 pr_debug("%s: no available resource selector found\n",
 264                          __func__);
 265                 ret = -ENOSPC;
 266                 goto out;
 267         }
 268
 269         /* Remember what counter we used */
 270         counter = 1 << ctridx;
 271
 272         /*
 273          * Initialise original and reload counter value to the smallest
 274          * possible value in order to get as much precision as we can.
 275          */
 276         config->cntr_val[ctridx] = 1;
 277         config->cntrldvr[ctridx] = 1;
 278
 279         /* Set the trace counter control register */
 280         val =  0x1 << 16        |  /* Bit 16, reload counter automatically */
 281                0x0 << 7         |  /* Select single resource selector */
 282                0x1;                /* Resource selector 1, i.e always true */
 283
 284         config->cntr_ctrl[ctridx] = val;
 285
 286         val = 0x2 << 16         | /* Group 0b0010 - Counter and sequencers */
 287               counter << 0;       /* Counter to use */
 288
 289         config->res_ctrl[rselector] = val;
 290
 291         val = 0x0 << 7          | /* Select single resource selector */
 292               rselector;          /* Resource selector */
 293
 294         config->ts_ctrl = val;
 295
 296         ret = 0;
 297 out:
 298         return ret;
 299 }
 300
 301 static int etm4_parse_event_config(struct etmv4_drvdata *drvdata,
 302                                    struct perf_event *event)
 303 {
 304         int ret = 0;
 305         struct etmv4_config *config = &drvdata->config;
 306         struct perf_event_attr *attr = &event->attr;
 307
 308         if (!attr) {
 309                 ret = -EINVAL;
 310                 goto out;
 311         }
 312
 313         /* Clear configuration from previous run */
 314         memset(config, 0, sizeof(struct etmv4_config));
 315
 316         if (attr->exclude_kernel)
 317                 config->mode = ETM_MODE_EXCL_KERN;
 318
 319         if (attr->exclude_user)
 320                 config->mode = ETM_MODE_EXCL_USER;
 321
 322         /* Always start from the default config */
 323         etm4_set_default_config(config);
 324
 325         /* Configure filters specified on the perf cmd line, if any. */
 326         ret = etm4_set_event_filters(drvdata, event);
 327         if (ret)
 328                 goto out;
 329
 330         /* Go from generic option to ETMv4 specifics */
 331         if (attr->config & BIT(ETM_OPT_CYCACC)) {
 332                 config->cfg |= BIT(4);
 333                 /* TRM: Must program this for cycacc to work */
 334                 config->ccctlr = ETM_CYC_THRESHOLD_DEFAULT;
 335         }
 336         if (attr->config & BIT(ETM_OPT_TS)) {
 337                 /*
 338                  * Configure timestamps to be emitted at regular intervals in
 339                  * order to correlate instructions executed on different CPUs
 340                  * (CPU-wide trace scenarios).
 341                  */
 342                 ret = etm4_config_timestamp_event(drvdata);
 343
 344                 /*
 345                  * No need to go further if timestamp intervals can't
 346                  * be configured.
 347                  */
 348                 if (ret)
 349                         goto out;
 350
 351                 /* bit[11], Global timestamp tracing bit */
 352                 config->cfg |= BIT(11);
 353         }
 354
 355         if (attr->config & BIT(ETM_OPT_CTXTID))
 356                 /* bit[6], Context ID tracing bit */
 357                 config->cfg |= BIT(ETM4_CFG_BIT_CTXTID);
 358
 359         /* return stack - enable if selected and supported */
 360         if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack)
 361                 /* bit[12], Return stack enable bit */
 362                 config->cfg |= BIT(12);
 363
 364 out:
 365         return ret;
 366 }
 367
 368 static int etm4_enable_perf(struct coresight_device *csdev,
 369                             struct perf_event *event)
 370 {
 371         int ret = 0;
 372         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 373
 374         if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) {
 375                 ret = -EINVAL;
 376                 goto out;
 377         }
 378
 379         /* Configure the tracer based on the session's specifics */
 380         ret = etm4_parse_event_config(drvdata, event);
 381         if (ret)
 382                 goto out;
 383         /* And enable it */
 384         ret = etm4_enable_hw(drvdata);
 385
 386 out:
 387         return ret;
 388 }
 389
 390 static int etm4_enable_sysfs(struct coresight_device *csdev)
 391 {
 392         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 393         struct etm4_enable_arg arg = { 0 };
 394         int ret;
 395
 396         spin_lock(&drvdata->spinlock);
 397
 398         /*
 399          * Executing etm4_enable_hw on the cpu whose ETM is being enabled
 400          * ensures that register writes occur when cpu is powered.
 401          */
 402         arg.drvdata = drvdata;
 403         ret = smp_call_function_single(drvdata->cpu,
 404                                        etm4_enable_hw_smp_call, &arg, 1);
 405         if (!ret)
 406                 ret = arg.rc;
 407         if (!ret)
 408                 drvdata->sticky_enable = true;
 409         spin_unlock(&drvdata->spinlock);
 410
 411         if (!ret)
 412                 dev_dbg(&csdev->dev, "ETM tracing enabled\n");
 413         return ret;
 414 }
 415
 416 static int etm4_enable(struct coresight_device *csdev,
 417                        struct perf_event *event, u32 mode)
 418 {
 419         int ret;
 420         u32 val;
 421         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 422
 423         val = local_cmpxchg(&drvdata->mode, CS_MODE_DISABLED, mode);
 424
 425         /* Someone is already using the tracer */
 426         if (val)
 427                 return -EBUSY;
 428
 429         switch (mode) {
 430         case CS_MODE_SYSFS:
 431                 ret = etm4_enable_sysfs(csdev);
 432                 break;
 433         case CS_MODE_PERF:
 434                 ret = etm4_enable_perf(csdev, event);
 435                 break;
 436         default:
 437                 ret = -EINVAL;
 438         }
 439
 440         /* The tracer didn't start */
 441         if (ret)
 442                 local_set(&drvdata->mode, CS_MODE_DISABLED);
 443
 444         return ret;
 445 }
 446
 447 static void etm4_disable_hw(void *info)
 448 {
 449         u32 control;
 450         struct etmv4_drvdata *drvdata = info;
 451
 452         CS_UNLOCK(drvdata->base);
 453
 454         /* power can be removed from the trace unit now */
 455         control = readl_relaxed(drvdata->base + TRCPDCR);
 456         control &= ~TRCPDCR_PU;
 457         writel_relaxed(control, drvdata->base + TRCPDCR);
 458
 459         control = readl_relaxed(drvdata->base + TRCPRGCTLR);
 460
 461         /* EN, bit[0] Trace unit enable bit */
 462         control &= ~0x1;
 463
 464         /*
 465          * Make sure everything completes before disabling, as recommended
 466          * by section 7.3.77 ("TRCVICTLR, ViewInst Main Control Register,
 467          * SSTATUS") of ARM IHI 0064D
 468          */
 469         dsb(sy);
 470         isb();
 471         writel_relaxed(control, drvdata->base + TRCPRGCTLR);
 472
 473         coresight_disclaim_device_unlocked(drvdata->base);
 474
 475         CS_LOCK(drvdata->base);
 476
 477         dev_dbg(&drvdata->csdev->dev,
 478                 "cpu: %d disable smp call done\n", drvdata->cpu);
 479 }
 480
 481 static int etm4_disable_perf(struct coresight_device *csdev,
 482                              struct perf_event *event)
 483 {
 484         u32 control;
 485         struct etm_filters *filters = event->hw.addr_filters;
 486         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 487
 488         if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id()))
 489                 return -EINVAL;
 490
 491         etm4_disable_hw(drvdata);
 492
 493         /*
 494          * Check if the start/stop logic was active when the unit was stopped.
 495          * That way we can re-enable the start/stop logic when the process is
 496          * scheduled again.  Configuration of the start/stop logic happens in
 497          * function etm4_set_event_filters().
 498          */
 499         control = readl_relaxed(drvdata->base + TRCVICTLR);
 500         /* TRCVICTLR::SSSTATUS, bit[9] */
 501         filters->ssstatus = (control & BIT(9));
 502
 503         return 0;
 504 }
 505
 506 static void etm4_disable_sysfs(struct coresight_device *csdev)
 507 {
 508         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 509
 510         /*
 511          * Taking hotplug lock here protects from clocks getting disabled
 512          * with tracing being left on (crash scenario) if user disable occurs
 513          * after cpu online mask indicates the cpu is offline but before the
 514          * DYING hotplug callback is serviced by the ETM driver.
 515          */
 516         cpus_read_lock();
 517         spin_lock(&drvdata->spinlock);
 518
 519         /*
 520          * Executing etm4_disable_hw on the cpu whose ETM is being disabled
 521          * ensures that register writes occur when cpu is powered.
 522          */
 523         smp_call_function_single(drvdata->cpu, etm4_disable_hw, drvdata, 1);
 524
 525         spin_unlock(&drvdata->spinlock);
 526         cpus_read_unlock();
 527
 528         dev_dbg(&csdev->dev, "ETM tracing disabled\n");
 529 }
 530
 531 static void etm4_disable(struct coresight_device *csdev,
 532                          struct perf_event *event)
 533 {
 534         u32 mode;
 535         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 536
 537         /*
 538          * For as long as the tracer isn't disabled another entity can't
 539          * change its status.  As such we can read the status here without
 540          * fearing it will change under us.
 541          */
 542         mode = local_read(&drvdata->mode);
 543
 544         switch (mode) {
 545         case CS_MODE_DISABLED:
 546                 break;
 547         case CS_MODE_SYSFS:
 548                 etm4_disable_sysfs(csdev);
 549                 break;
 550         case CS_MODE_PERF:
 551                 etm4_disable_perf(csdev, event);
 552                 break;
 553         }
 554
 555         if (mode)
 556                 local_set(&drvdata->mode, CS_MODE_DISABLED);
 557 }
 558
 559 static const struct coresight_ops_source etm4_source_ops = {
 560         .cpu_id         = etm4_cpu_id,
 561         .trace_id       = etm4_trace_id,
 562         .enable         = etm4_enable,
 563         .disable        = etm4_disable,
 564 };
 565
 566 static const struct coresight_ops etm4_cs_ops = {
 567         .source_ops     = &etm4_source_ops,
 568 };
 569
 570 static void etm4_init_arch_data(void *info)
 571 {
 572         u32 etmidr0;
 573         u32 etmidr1;
 574         u32 etmidr2;
 575         u32 etmidr3;
 576         u32 etmidr4;
 577         u32 etmidr5;
 578         struct etmv4_drvdata *drvdata = info;
 579
 580         /* Make sure all registers are accessible */
 581         etm4_os_unlock(drvdata);
 582
 583         CS_UNLOCK(drvdata->base);
 584
 585         /* find all capabilities of the tracing unit */
 586         etmidr0 = readl_relaxed(drvdata->base + TRCIDR0);
 587
 588         /* INSTP0, bits[2:1] P0 tracing support field */
 589         if (BMVAL(etmidr0, 1, 1) && BMVAL(etmidr0, 2, 2))
 590                 drvdata->instrp0 = true;
 591         else
 592                 drvdata->instrp0 = false;
 593
 594         /* TRCBB, bit[5] Branch broadcast tracing support bit */
 595         if (BMVAL(etmidr0, 5, 5))
 596                 drvdata->trcbb = true;
 597         else
 598                 drvdata->trcbb = false;
 599
 600         /* TRCCOND, bit[6] Conditional instruction tracing support bit */
 601         if (BMVAL(etmidr0, 6, 6))
 602                 drvdata->trccond = true;
 603         else
 604                 drvdata->trccond = false;
 605
 606         /* TRCCCI, bit[7] Cycle counting instruction bit */
 607         if (BMVAL(etmidr0, 7, 7))
 608                 drvdata->trccci = true;
 609         else
 610                 drvdata->trccci = false;
 611
 612         /* RETSTACK, bit[9] Return stack bit */
 613         if (BMVAL(etmidr0, 9, 9))
 614                 drvdata->retstack = true;
 615         else
 616                 drvdata->retstack = false;
 617
 618         /* NUMEVENT, bits[11:10] Number of events field */
 619         drvdata->nr_event = BMVAL(etmidr0, 10, 11);
 620         /* QSUPP, bits[16:15] Q element support field */
 621         drvdata->q_support = BMVAL(etmidr0, 15, 16);
 622         /* TSSIZE, bits[28:24] Global timestamp size field */
 623         drvdata->ts_size = BMVAL(etmidr0, 24, 28);
 624
 625         /* base architecture of trace unit */
 626         etmidr1 = readl_relaxed(drvdata->base + TRCIDR1);
 627         /*
 628          * TRCARCHMIN, bits[7:4] architecture the minor version number
 629          * TRCARCHMAJ, bits[11:8] architecture major versin number
 630          */
 631         drvdata->arch = BMVAL(etmidr1, 4, 11);
 632
 633         /* maximum size of resources */
 634         etmidr2 = readl_relaxed(drvdata->base + TRCIDR2);
 635         /* CIDSIZE, bits[9:5] Indicates the Context ID size */
 636         drvdata->ctxid_size = BMVAL(etmidr2, 5, 9);
 637         /* VMIDSIZE, bits[14:10] Indicates the VMID size */
 638         drvdata->vmid_size = BMVAL(etmidr2, 10, 14);
 639         /* CCSIZE, bits[28:25] size of the cycle counter in bits minus 12 */
 640         drvdata->ccsize = BMVAL(etmidr2, 25, 28);
 641
 642         etmidr3 = readl_relaxed(drvdata->base + TRCIDR3);
 643         /* CCITMIN, bits[11:0] minimum threshold value that can be programmed */
 644         drvdata->ccitmin = BMVAL(etmidr3, 0, 11);
 645         /* EXLEVEL_S, bits[19:16] Secure state instruction tracing */
 646         drvdata->s_ex_level = BMVAL(etmidr3, 16, 19);
 647         /* EXLEVEL_NS, bits[23:20] Non-secure state instruction tracing */
 648         drvdata->ns_ex_level = BMVAL(etmidr3, 20, 23);
 649
 650         /*
 651          * TRCERR, bit[24] whether a trace unit can trace a
 652          * system error exception.
 653          */
 654         if (BMVAL(etmidr3, 24, 24))
 655                 drvdata->trc_error = true;
 656         else
 657                 drvdata->trc_error = false;
 658
 659         /* SYNCPR, bit[25] implementation has a fixed synchronization period? */
 660         if (BMVAL(etmidr3, 25, 25))
 661                 drvdata->syncpr = true;
 662         else
 663                 drvdata->syncpr = false;
 664
 665         /* STALLCTL, bit[26] is stall control implemented? */
 666         if (BMVAL(etmidr3, 26, 26))
 667                 drvdata->stallctl = true;
 668         else
 669                 drvdata->stallctl = false;
 670
 671         /* SYSSTALL, bit[27] implementation can support stall control? */
 672         if (BMVAL(etmidr3, 27, 27))
 673                 drvdata->sysstall = true;
 674         else
 675                 drvdata->sysstall = false;
 676
 677         /* NUMPROC, bits[30:28] the number of PEs available for tracing */
 678         drvdata->nr_pe = BMVAL(etmidr3, 28, 30);
 679
 680         /* NOOVERFLOW, bit[31] is trace overflow prevention supported */
 681         if (BMVAL(etmidr3, 31, 31))
 682                 drvdata->nooverflow = true;
 683         else
 684                 drvdata->nooverflow = false;
 685
 686         /* number of resources trace unit supports */
 687         etmidr4 = readl_relaxed(drvdata->base + TRCIDR4);
 688         /* NUMACPAIRS, bits[0:3] number of addr comparator pairs for tracing */
 689         drvdata->nr_addr_cmp = BMVAL(etmidr4, 0, 3);
 690         /* NUMPC, bits[15:12] number of PE comparator inputs for tracing */
 691         drvdata->nr_pe_cmp = BMVAL(etmidr4, 12, 15);
 692         /*
 693          * NUMRSPAIR, bits[19:16]
 694          * The number of resource pairs conveyed by the HW starts at 0, i.e a
 695          * value of 0x0 indicate 1 resource pair, 0x1 indicate two and so on.
 696          * As such add 1 to the value of NUMRSPAIR for a better representation.
 697          */
 698         drvdata->nr_resource = BMVAL(etmidr4, 16, 19) + 1;
 699         /*
 700          * NUMSSCC, bits[23:20] the number of single-shot
 701          * comparator control for tracing
 702          */
 703         drvdata->nr_ss_cmp = BMVAL(etmidr4, 20, 23);
 704         /* NUMCIDC, bits[27:24] number of Context ID comparators for tracing */
 705         drvdata->numcidc = BMVAL(etmidr4, 24, 27);
 706         /* NUMVMIDC, bits[31:28] number of VMID comparators for tracing */
 707         drvdata->numvmidc = BMVAL(etmidr4, 28, 31);
 708
 709         etmidr5 = readl_relaxed(drvdata->base + TRCIDR5);
 710         /* NUMEXTIN, bits[8:0] number of external inputs implemented */
 711         drvdata->nr_ext_inp = BMVAL(etmidr5, 0, 8);
 712         /* TRACEIDSIZE, bits[21:16] indicates the trace ID width */
 713         drvdata->trcid_size = BMVAL(etmidr5, 16, 21);
 714         /* ATBTRIG, bit[22] implementation can support ATB triggers? */
 715         if (BMVAL(etmidr5, 22, 22))
 716                 drvdata->atbtrig = true;
 717         else
 718                 drvdata->atbtrig = false;
 719         /*
 720          * LPOVERRIDE, bit[23] implementation supports
 721          * low-power state override
 722          */
 723         if (BMVAL(etmidr5, 23, 23))
 724                 drvdata->lpoverride = true;
 725         else
 726                 drvdata->lpoverride = false;
 727         /* NUMSEQSTATE, bits[27:25] number of sequencer states implemented */
 728         drvdata->nrseqstate = BMVAL(etmidr5, 25, 27);
 729         /* NUMCNTR, bits[30:28] number of counters available for tracing */
 730         drvdata->nr_cntr = BMVAL(etmidr5, 28, 30);
 731         CS_LOCK(drvdata->base);
 732 }
 733
 734 static void etm4_set_default_config(struct etmv4_config *config)
 735 {
 736         /* disable all events tracing */
 737         config->eventctrl0 = 0x0;
 738         config->eventctrl1 = 0x0;
 739
 740         /* disable stalling */
 741         config->stall_ctrl = 0x0;
 742
 743         /* enable trace synchronization every 4096 bytes, if available */
 744         config->syncfreq = 0xC;
 745
 746         /* disable timestamp event */
 747         config->ts_ctrl = 0x0;
 748
 749         /* TRCVICTLR::EVENT = 0x01, select the always on logic */
 750         config->vinst_ctrl |= BIT(0);
 751 }
 752
 753 static u64 etm4_get_ns_access_type(struct etmv4_config *config)
 754 {
 755         u64 access_type = 0;
 756
 757         /*
 758          * EXLEVEL_NS, bits[15:12]
 759          * The Exception levels are:
 760          *   Bit[12] Exception level 0 - Application
 761          *   Bit[13] Exception level 1 - OS
 762          *   Bit[14] Exception level 2 - Hypervisor
 763          *   Bit[15] Never implemented
 764          */
 765         if (!is_kernel_in_hyp_mode()) {
 766                 /* Stay away from hypervisor mode for non-VHE */
 767                 access_type =  ETM_EXLEVEL_NS_HYP;
 768                 if (config->mode & ETM_MODE_EXCL_KERN)
 769                         access_type |= ETM_EXLEVEL_NS_OS;
 770         } else if (config->mode & ETM_MODE_EXCL_KERN) {
 771                 access_type = ETM_EXLEVEL_NS_HYP;
 772         }
 773
 774         if (config->mode & ETM_MODE_EXCL_USER)
 775                 access_type |= ETM_EXLEVEL_NS_APP;
 776
 777         return access_type;
 778 }
 779
 780 static u64 etm4_get_access_type(struct etmv4_config *config)
 781 {
 782         u64 access_type = etm4_get_ns_access_type(config);
 783
 784         /*
 785          * EXLEVEL_S, bits[11:8], don't trace anything happening
 786          * in secure state.
 787          */
 788         access_type |= (ETM_EXLEVEL_S_APP       |
 789                         ETM_EXLEVEL_S_OS        |
 790                         ETM_EXLEVEL_S_HYP);
 791
 792         return access_type;
 793 }
 794
 795 static void etm4_set_comparator_filter(struct etmv4_config *config,
 796                                        u64 start, u64 stop, int comparator)
 797 {
 798         u64 access_type = etm4_get_access_type(config);
 799
 800         /* First half of default address comparator */
 801         config->addr_val[comparator] = start;
 802         config->addr_acc[comparator] = access_type;
 803         config->addr_type[comparator] = ETM_ADDR_TYPE_RANGE;
 804
 805         /* Second half of default address comparator */
 806         config->addr_val[comparator + 1] = stop;
 807         config->addr_acc[comparator + 1] = access_type;
 808         config->addr_type[comparator + 1] = ETM_ADDR_TYPE_RANGE;
 809
 810         /*
 811          * Configure the ViewInst function to include this address range
 812          * comparator.
 813          *
 814          * @comparator is divided by two since it is the index in the
 815          * etmv4_config::addr_val array but register TRCVIIECTLR deals with
 816          * address range comparator _pairs_.
 817          *
 818          * Therefore:
 819          *      index 0 -> compatator pair 0
 820          *      index 2 -> comparator pair 1
 821          *      index 4 -> comparator pair 2
 822          *      ...
 823          *      index 14 -> comparator pair 7
 824          */
 825         config->viiectlr |= BIT(comparator / 2);
 826 }
 827
 828 static void etm4_set_start_stop_filter(struct etmv4_config *config,
 829                                        u64 address, int comparator,
 830                                        enum etm_addr_type type)
 831 {
 832         int shift;
 833         u64 access_type = etm4_get_access_type(config);
 834
 835         /* Configure the comparator */
 836         config->addr_val[comparator] = address;
 837         config->addr_acc[comparator] = access_type;
 838         config->addr_type[comparator] = type;
 839
 840         /*
 841          * Configure ViewInst Start-Stop control register.
 842          * Addresses configured to start tracing go from bit 0 to n-1,
 843          * while those configured to stop tracing from 16 to 16 + n-1.
 844          */
 845         shift = (type == ETM_ADDR_TYPE_START ? 0 : 16);
 846         config->vissctlr |= BIT(shift + comparator);
 847 }
 848
 849 static void etm4_set_default_filter(struct etmv4_config *config)
 850 {
 851         u64 start, stop;
 852
 853         /*
 854          * Configure address range comparator '0' to encompass all
 855          * possible addresses.
 856          */
 857         start = 0x0;
 858         stop = ~0x0;
 859
 860         etm4_set_comparator_filter(config, start, stop,
 861                                    ETM_DEFAULT_ADDR_COMP);
 862
 863         /*
 864          * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
 865          * in the started state
 866          */
 867         config->vinst_ctrl |= BIT(9);
 868
 869         /* No start-stop filtering for ViewInst */
 870         config->vissctlr = 0x0;
 871 }
 872
 873 static void etm4_set_default(struct etmv4_config *config)
 874 {
 875         if (WARN_ON_ONCE(!config))
 876                 return;
 877
 878         /*
 879          * Make default initialisation trace everything
 880          *
 881          * Select the "always true" resource selector on the
 882          * "Enablign Event" line and configure address range comparator
 883          * '0' to trace all the possible address range.  From there
 884          * configure the "include/exclude" engine to include address
 885          * range comparator '0'.
 886          */
 887         etm4_set_default_config(config);
 888         etm4_set_default_filter(config);
 889 }
 890
 891 static int etm4_get_next_comparator(struct etmv4_drvdata *drvdata, u32 type)
 892 {
 893         int nr_comparator, index = 0;
 894         struct etmv4_config *config = &drvdata->config;
 895
 896         /*
 897          * nr_addr_cmp holds the number of comparator _pair_, so time 2
 898          * for the total number of comparators.
 899          */
 900         nr_comparator = drvdata->nr_addr_cmp * 2;
 901
 902         /* Go through the tally of comparators looking for a free one. */
 903         while (index < nr_comparator) {
 904                 switch (type) {
 905                 case ETM_ADDR_TYPE_RANGE:
 906                         if (config->addr_type[index] == ETM_ADDR_TYPE_NONE &&
 907                             config->addr_type[index + 1] == ETM_ADDR_TYPE_NONE)
 908                                 return index;
 909
 910                         /* Address range comparators go in pairs */
 911                         index += 2;
 912                         break;
 913                 case ETM_ADDR_TYPE_START:
 914                 case ETM_ADDR_TYPE_STOP:
 915                         if (config->addr_type[index] == ETM_ADDR_TYPE_NONE)
 916                                 return index;
 917
 918                         /* Start/stop address can have odd indexes */
 919                         index += 1;
 920                         break;
 921                 default:
 922                         return -EINVAL;
 923                 }
 924         }
 925
 926         /* If we are here all the comparators have been used. */
 927         return -ENOSPC;
 928 }
 929
 930 static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
 931                                   struct perf_event *event)
 932 {
 933         int i, comparator, ret = 0;
 934         u64 address;
 935         struct etmv4_config *config = &drvdata->config;
 936         struct etm_filters *filters = event->hw.addr_filters;
 937
 938         if (!filters)
 939                 goto default_filter;
 940
 941         /* Sync events with what Perf got */
 942         perf_event_addr_filters_sync(event);
 943
 944         /*
 945          * If there are no filters to deal with simply go ahead with
 946          * the default filter, i.e the entire address range.
 947          */
 948         if (!filters->nr_filters)
 949                 goto default_filter;
 950
 951         for (i = 0; i < filters->nr_filters; i++) {
 952                 struct etm_filter *filter = &filters->etm_filter[i];
 953                 enum etm_addr_type type = filter->type;
 954
 955                 /* See if a comparator is free. */
 956                 comparator = etm4_get_next_comparator(drvdata, type);
 957                 if (comparator < 0) {
 958                         ret = comparator;
 959                         goto out;
 960                 }
 961
 962                 switch (type) {
 963                 case ETM_ADDR_TYPE_RANGE:
 964                         etm4_set_comparator_filter(config,
 965                                                    filter->start_addr,
 966                                                    filter->stop_addr,
 967                                                    comparator);
 968                         /*
 969                          * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
 970                          * in the started state
 971                          */
 972                         config->vinst_ctrl |= BIT(9);
 973
 974                         /* No start-stop filtering for ViewInst */
 975                         config->vissctlr = 0x0;
 976                         break;
 977                 case ETM_ADDR_TYPE_START:
 978                 case ETM_ADDR_TYPE_STOP:
 979                         /* Get the right start or stop address */
 980                         address = (type == ETM_ADDR_TYPE_START ?
 981                                    filter->start_addr :
 982                                    filter->stop_addr);
 983
 984                         /* Configure comparator */
 985                         etm4_set_start_stop_filter(config, address,
 986                                                    comparator, type);
 987
 988                         /*
 989                          * If filters::ssstatus == 1, trace acquisition was
 990                          * started but the process was yanked away before the
 991                          * the stop address was hit.  As such the start/stop
 992                          * logic needs to be re-started so that tracing can
 993                          * resume where it left.
 994                          *
 995                          * The start/stop logic status when a process is
 996                          * scheduled out is checked in function
 997                          * etm4_disable_perf().
 998                          */
 999                         if (filters->ssstatus)
1000                                 config->vinst_ctrl |= BIT(9);
1001
1002                         /* No include/exclude filtering for ViewInst */
1003                         config->viiectlr = 0x0;
1004                         break;
1005                 default:
1006                         ret = -EINVAL;
1007                         goto out;
1008                 }
1009         }
1010
1011         goto out;
1012
1013
1014 default_filter:
1015         etm4_set_default_filter(config);
1016
1017 out:
1018         return ret;
1019 }
1020
1021 void etm4_config_trace_mode(struct etmv4_config *config)
1022 {
1023         u32 addr_acc, mode;
1024
1025         mode = config->mode;
1026         mode &= (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER);
1027
1028         /* excluding kernel AND user space doesn't make sense */
1029         WARN_ON_ONCE(mode == (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER));
1030
1031         /* nothing to do if neither flags are set */
1032         if (!(mode & ETM_MODE_EXCL_KERN) && !(mode & ETM_MODE_EXCL_USER))
1033                 return;
1034
1035         addr_acc = config->addr_acc[ETM_DEFAULT_ADDR_COMP];
1036         /* clear default config */
1037         addr_acc &= ~(ETM_EXLEVEL_NS_APP | ETM_EXLEVEL_NS_OS |
1038                       ETM_EXLEVEL_NS_HYP);
1039
1040         addr_acc |= etm4_get_ns_access_type(config);
1041
1042         config->addr_acc[ETM_DEFAULT_ADDR_COMP] = addr_acc;
1043         config->addr_acc[ETM_DEFAULT_ADDR_COMP + 1] = addr_acc;
1044 }
1045
1046 static int etm4_online_cpu(unsigned int cpu)
1047 {
1048         if (!etmdrvdata[cpu])
1049                 return 0;
1050
1051         if (etmdrvdata[cpu]->boot_enable && !etmdrvdata[cpu]->sticky_enable)
1052                 coresight_enable(etmdrvdata[cpu]->csdev);
1053         return 0;
1054 }
1055
1056 static int etm4_starting_cpu(unsigned int cpu)
1057 {
1058         if (!etmdrvdata[cpu])
1059                 return 0;
1060
1061         spin_lock(&etmdrvdata[cpu]->spinlock);
1062         if (!etmdrvdata[cpu]->os_unlock)
1063                 etm4_os_unlock(etmdrvdata[cpu]);
1064
1065         if (local_read(&etmdrvdata[cpu]->mode))
1066                 etm4_enable_hw(etmdrvdata[cpu]);
1067         spin_unlock(&etmdrvdata[cpu]->spinlock);
1068         return 0;
1069 }
1070
1071 static int etm4_dying_cpu(unsigned int cpu)
1072 {
1073         if (!etmdrvdata[cpu])
1074                 return 0;
1075
1076         spin_lock(&etmdrvdata[cpu]->spinlock);
1077         if (local_read(&etmdrvdata[cpu]->mode))
1078                 etm4_disable_hw(etmdrvdata[cpu]);
1079         spin_unlock(&etmdrvdata[cpu]->spinlock);
1080         return 0;
1081 }
1082
1083 static void etm4_init_trace_id(struct etmv4_drvdata *drvdata)
1084 {
1085         drvdata->trcid = coresight_get_trace_id(drvdata->cpu);
1086 }
1087
1088 static int etm4_probe(struct amba_device *adev, const struct amba_id *id)
1089 {
1090         int ret;
1091         void __iomem *base;
1092         struct device *dev = &adev->dev;
1093         struct coresight_platform_data *pdata = NULL;
1094         struct etmv4_drvdata *drvdata;
1095         struct resource *res = &adev->res;
1096         struct coresight_desc desc = { 0 };
1097
1098         drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
1099         if (!drvdata)
1100                 return -ENOMEM;
1101
1102         dev_set_drvdata(dev, drvdata);
1103
1104         /* Validity for the resource is already checked by the AMBA core */
1105         base = devm_ioremap_resource(dev, res);
1106         if (IS_ERR(base))
1107                 return PTR_ERR(base);
1108
1109         drvdata->base = base;
1110
1111         spin_lock_init(&drvdata->spinlock);
1112
1113         drvdata->cpu = coresight_get_cpu(dev);
1114         if (drvdata->cpu < 0)
1115                 return drvdata->cpu;
1116
1117         desc.name = devm_kasprintf(dev, GFP_KERNEL, "etm%d", drvdata->cpu);
1118         if (!desc.name)
1119                 return -ENOMEM;
1120
1121         cpus_read_lock();
1122         etmdrvdata[drvdata->cpu] = drvdata;
1123
1124         if (smp_call_function_single(drvdata->cpu,
1125                                 etm4_init_arch_data,  drvdata, 1))
1126                 dev_err(dev, "ETM arch init failed\n");
1127
1128         if (!etm4_count++) {
1129                 cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ARM_CORESIGHT_STARTING,
1130                                                      "arm/coresight4:starting",
1131                                                      etm4_starting_cpu, etm4_dying_cpu);
1132                 ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
1133                                                            "arm/coresight4:online",
1134                                                            etm4_online_cpu, NULL);
1135                 if (ret < 0)
1136                         goto err_arch_supported;
1137                 hp_online = ret;
1138         }
1139
1140         cpus_read_unlock();
1141
1142         if (etm4_arch_supported(drvdata->arch) == false) {
1143                 ret = -EINVAL;
1144                 goto err_arch_supported;
1145         }
1146
1147         etm4_init_trace_id(drvdata);
1148         etm4_set_default(&drvdata->config);
1149
1150         pdata = coresight_get_platform_data(dev);
1151         if (IS_ERR(pdata)) {
1152                 ret = PTR_ERR(pdata);
1153                 goto err_arch_supported;
1154         }
1155         adev->dev.platform_data = pdata;
1156
1157         desc.type = CORESIGHT_DEV_TYPE_SOURCE;
1158         desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
1159         desc.ops = &etm4_cs_ops;
1160         desc.pdata = pdata;
1161         desc.dev = dev;
1162         desc.groups = coresight_etmv4_groups;
1163         drvdata->csdev = coresight_register(&desc);
1164         if (IS_ERR(drvdata->csdev)) {
1165                 ret = PTR_ERR(drvdata->csdev);
1166                 goto err_arch_supported;
1167         }
1168
1169         ret = etm_perf_symlink(drvdata->csdev, true);
1170         if (ret) {
1171                 coresight_unregister(drvdata->csdev);
1172                 goto err_arch_supported;
1173         }
1174
1175         pm_runtime_put(&adev->dev);
1176         dev_info(&drvdata->csdev->dev, "CPU%d: ETM v%d.%d initialized\n",
1177                  drvdata->cpu, drvdata->arch >> 4, drvdata->arch & 0xf);
1178
1179         if (boot_enable) {
1180                 coresight_enable(drvdata->csdev);
1181                 drvdata->boot_enable = true;
1182         }
1183
1184         return 0;
1185
1186 err_arch_supported:
1187         if (--etm4_count == 0) {
1188                 cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
1189                 if (hp_online)
1190                         cpuhp_remove_state_nocalls(hp_online);
1191         }
1192         return ret;
1193 }
1194
1195 static struct amba_cs_uci_id uci_id_etm4[] = {
1196         {
1197                 /*  ETMv4 UCI data */
1198                 .devarch        = 0x47704a13,
1199                 .devarch_mask   = 0xfff0ffff,
1200                 .devtype        = 0x00000013,
1201         }
1202 };
1203
1204 static const struct amba_id etm4_ids[] = {
1205         CS_AMBA_ID(0x000bb95d),                 /* Cortex-A53 */
1206         CS_AMBA_ID(0x000bb95e),                 /* Cortex-A57 */
1207         CS_AMBA_ID(0x000bb95a),                 /* Cortex-A72 */
1208         CS_AMBA_ID(0x000bb959),                 /* Cortex-A73 */
1209         CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),/* Cortex-A35 */
1210         CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),/* Qualcomm Kryo */
1211         CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),/* Qualcomm Kryo */
1212         CS_AMBA_ID(0x000bb802),                 /* Qualcomm Kryo 385 Cortex-A55 */
1213         CS_AMBA_ID(0x000bb803),                 /* Qualcomm Kryo 385 Cortex-A75 */
1214         {},
1215 };
1216
1217 static struct amba_driver etm4x_driver = {
1218         .drv = {
1219                 .name   = "coresight-etm4x",
1220                 .suppress_bind_attrs = true,
1221         },
1222         .probe          = etm4_probe,
1223         .id_table       = etm4_ids,
1224 };
1225 builtin_amba_driver(etm4x_driver);