drivers/staging/gasket/gasket_interrupt.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (C) 2018 Google, Inc. */
   3
   4 #include "gasket_interrupt.h"
   5
   6 #include "gasket_constants.h"
   7 #include "gasket_core.h"
   8 #include "gasket_sysfs.h"
   9 #include <linux/device.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/printk.h>
  12 #include <linux/version.h>
  13 #ifdef GASKET_KERNEL_TRACE_SUPPORT
  14 #define CREATE_TRACE_POINTS
  15 #include <trace/events/gasket_interrupt.h>
  16 #else
  17 #define trace_gasket_interrupt_event(x, ...)
  18 #endif
  19 /* Retry attempts if the requested number of interrupts aren't available. */
  20 #define MSIX_RETRY_COUNT 3
  21
  22 /* Instance interrupt management data. */
  23 struct gasket_interrupt_data {
  24         /* The name associated with this interrupt data. */
  25         const char *name;
  26
  27         /* Interrupt type. See gasket_interrupt_type in gasket_core.h */
  28         int type;
  29
  30         /* The PCI device [if any] associated with the owning device. */
  31         struct pci_dev *pci_dev;
  32
  33         /* Set to 1 if MSI-X has successfully been configred, 0 otherwise. */
  34         int msix_configured;
  35
  36         /* The number of interrupts requested by the owning device. */
  37         int num_interrupts;
  38
  39         /* A pointer to the interrupt descriptor struct for this device. */
  40         const struct gasket_interrupt_desc *interrupts;
  41
  42         /* The index of the bar into which interrupts should be mapped. */
  43         int interrupt_bar_index;
  44
  45         /* The width of a single interrupt in a packed interrupt register. */
  46         int pack_width;
  47
  48         /* offset of wire interrupt registers */
  49         const struct gasket_wire_interrupt_offsets *wire_interrupt_offsets;
  50
  51         /*
  52          * Design-wise, these elements should be bundled together, but
  53          * pci_enable_msix's interface requires that they be managed
  54          * individually (requires array of struct msix_entry).
  55          */
  56
  57         /* The number of successfully configured interrupts. */
  58         int num_configured;
  59
  60         /* The MSI-X data for each requested/configured interrupt. */
  61         struct msix_entry *msix_entries;
  62
  63         /* The eventfd "callback" data for each interrupt. */
  64         struct eventfd_ctx **eventfd_ctxs;
  65
  66         /* The number of times each interrupt has been called. */
  67         ulong *interrupt_counts;
  68
  69         /* Linux IRQ number. */
  70         int irq;
  71 };
  72
  73 /* Structures to display interrupt counts in sysfs. */
  74 enum interrupt_sysfs_attribute_type {
  75         ATTR_INTERRUPT_COUNTS,
  76 };
  77
  78 /* Set up device registers for interrupt handling. */
  79 static void gasket_interrupt_setup(struct gasket_dev *gasket_dev)
  80 {
  81         int i;
  82         int pack_shift;
  83         ulong mask;
  84         ulong value;
  85         struct gasket_interrupt_data *interrupt_data =
  86                 gasket_dev->interrupt_data;
  87
  88         if (!interrupt_data) {
  89                 dev_dbg(gasket_dev->dev, "Interrupt data is not initialized\n");
  90                 return;
  91         }
  92
  93         dev_dbg(gasket_dev->dev, "Running interrupt setup\n");
  94
  95         if (interrupt_data->type == PLATFORM_WIRE ||
  96             interrupt_data->type == PCI_MSI) {
  97                 /* Nothing needs to be done for platform or PCI devices. */
  98                 return;
  99         }
 100
 101         if (interrupt_data->type != PCI_MSIX) {
 102                 dev_dbg(gasket_dev->dev,
 103                         "Cannot handle unsupported interrupt type %d\n",
 104                         interrupt_data->type);
 105                 return;
 106         }
 107
 108         /* Setup the MSIX table. */
 109
 110         for (i = 0; i < interrupt_data->num_interrupts; i++) {
 111                 /*
 112                  * If the interrupt is not packed, we can write the index into
 113                  * the register directly. If not, we need to deal with a read-
 114                  * modify-write and shift based on the packing index.
 115                  */
 116                 dev_dbg(gasket_dev->dev,
 117                         "Setting up interrupt index %d with index 0x%llx and "
 118                         "packing %d\n",
 119                         interrupt_data->interrupts[i].index,
 120                         interrupt_data->interrupts[i].reg,
 121                         interrupt_data->interrupts[i].packing);
 122                 if (interrupt_data->interrupts[i].packing == UNPACKED) {
 123                         value = interrupt_data->interrupts[i].index;
 124                 } else {
 125                         switch (interrupt_data->interrupts[i].packing) {
 126                         case PACK_0:
 127                                 pack_shift = 0;
 128                                 break;
 129                         case PACK_1:
 130                                 pack_shift = interrupt_data->pack_width;
 131                                 break;
 132                         case PACK_2:
 133                                 pack_shift = 2 * interrupt_data->pack_width;
 134                                 break;
 135                         case PACK_3:
 136                                 pack_shift = 3 * interrupt_data->pack_width;
 137                                 break;
 138                         default:
 139                                 dev_dbg(gasket_dev->dev,
 140                                         "Found interrupt description with "
 141                                         "unknown enum %d\n",
 142                                         interrupt_data->interrupts[i].packing);
 143                                 return;
 144                         }
 145
 146                         mask = ~(0xFFFF << pack_shift);
 147                         value = gasket_dev_read_64(gasket_dev,
 148                                                    interrupt_data->interrupt_bar_index,
 149                                                    interrupt_data->interrupts[i].reg);
 150                         value &= mask;
 151                         value |= interrupt_data->interrupts[i].index
 152                                  << pack_shift;
 153                 }
 154                 gasket_dev_write_64(gasket_dev, value,
 155                                     interrupt_data->interrupt_bar_index,
 156                                     interrupt_data->interrupts[i].reg);
 157         }
 158 }
 159
 160 static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id)
 161 {
 162         struct eventfd_ctx *ctx;
 163         struct gasket_interrupt_data *interrupt_data = dev_id;
 164         int interrupt = -1;
 165         int i;
 166
 167         /* If this linear lookup is a problem, we can maintain a map/hash. */
 168         for (i = 0; i < interrupt_data->num_interrupts; i++) {
 169                 if (interrupt_data->msix_entries[i].vector == irq) {
 170                         interrupt = interrupt_data->msix_entries[i].entry;
 171                         break;
 172                 }
 173         }
 174         if (interrupt == -1) {
 175                 pr_err("Received unknown irq %d\n", irq);
 176                 return IRQ_HANDLED;
 177         }
 178         trace_gasket_interrupt_event(interrupt_data->name, interrupt);
 179
 180         ctx = interrupt_data->eventfd_ctxs[interrupt];
 181         if (ctx)
 182                 eventfd_signal(ctx, 1);
 183
 184         ++(interrupt_data->interrupt_counts[interrupt]);
 185
 186         return IRQ_HANDLED;
 187 }
 188
 189 static int
 190 gasket_interrupt_msix_init(struct gasket_interrupt_data *interrupt_data)
 191 {
 192         int ret = 1;
 193         int i;
 194
 195         for (i = 0; i < interrupt_data->num_interrupts; i++) {
 196                 interrupt_data->msix_entries[i].entry = i;
 197                 interrupt_data->msix_entries[i].vector = 0;
 198                 interrupt_data->eventfd_ctxs[i] = NULL;
 199         }
 200
 201         /* Retry MSIX_RETRY_COUNT times if not enough IRQs are available. */
 202         for (i = 0; i < MSIX_RETRY_COUNT && ret > 0; i++)
 203                 ret = pci_enable_msix_exact(interrupt_data->pci_dev,
 204                                             interrupt_data->msix_entries,
 205                                             interrupt_data->num_interrupts);
 206
 207         if (ret)
 208                 return ret > 0 ? -EBUSY : ret;
 209         interrupt_data->msix_configured = 1;
 210
 211         for (i = 0; i < interrupt_data->num_interrupts; i++) {
 212                 ret = request_irq(interrupt_data->msix_entries[i].vector,
 213                                   gasket_msix_interrupt_handler, 0,
 214                                   interrupt_data->name, interrupt_data);
 215
 216                 if (ret) {
 217                         dev_err(&interrupt_data->pci_dev->dev,
 218                                 "Cannot get IRQ for interrupt %d, vector %d; "
 219                                 "%d\n",
 220                                 i, interrupt_data->msix_entries[i].vector, ret);
 221                         return ret;
 222                 }
 223
 224                 interrupt_data->num_configured++;
 225         }
 226
 227         return 0;
 228 }
 229
 230 /*
 231  * On QCM DragonBoard, we exit gasket_interrupt_msix_init() and kernel interrupt
 232  * setup code with MSIX vectors masked. This is wrong because nothing else in
 233  * the driver will normally touch the MSIX vectors.
 234  *
 235  * As a temporary hack, force unmasking there.
 236  *
 237  * TODO: Figure out why QCM kernel doesn't unmask the MSIX vectors, after
 238  * gasket_interrupt_msix_init(), and remove this code.
 239  */
 240 static void force_msix_interrupt_unmasking(struct gasket_dev *gasket_dev)
 241 {
 242         int i;
 243 #define MSIX_VECTOR_SIZE 16
 244 #define MSIX_MASK_BIT_OFFSET 12
 245 #define APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE 0x46800
 246         for (i = 0; i < gasket_dev->interrupt_data->num_configured; i++) {
 247                 /* Check if the MSIX vector is unmasked */
 248                 ulong location = APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE +
 249                                  MSIX_MASK_BIT_OFFSET + i * MSIX_VECTOR_SIZE;
 250                 u32 mask =
 251                         gasket_dev_read_32(gasket_dev,
 252                                            gasket_dev->interrupt_data->interrupt_bar_index,
 253                                            location);
 254                 if (!(mask & 1))
 255                         continue;
 256                 /* Unmask the msix vector (clear 32 bits) */
 257                 gasket_dev_write_32(gasket_dev, 0,
 258                                     gasket_dev->interrupt_data->interrupt_bar_index,
 259                                     location);
 260         }
 261 #undef MSIX_VECTOR_SIZE
 262 #undef MSIX_MASK_BIT_OFFSET
 263 #undef APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE
 264 }
 265
 266 static ssize_t interrupt_sysfs_show(struct device *device,
 267                                     struct device_attribute *attr, char *buf)
 268 {
 269         int i, ret;
 270         ssize_t written = 0, total_written = 0;
 271         struct gasket_interrupt_data *interrupt_data;
 272         struct gasket_dev *gasket_dev;
 273         struct gasket_sysfs_attribute *gasket_attr;
 274         enum interrupt_sysfs_attribute_type sysfs_type;
 275
 276         gasket_dev = gasket_sysfs_get_device_data(device);
 277         if (!gasket_dev) {
 278                 dev_dbg(device, "No sysfs mapping found for device\n");
 279                 return 0;
 280         }
 281
 282         gasket_attr = gasket_sysfs_get_attr(device, attr);
 283         if (!gasket_attr) {
 284                 dev_dbg(device, "No sysfs attr data found for device\n");
 285                 gasket_sysfs_put_device_data(device, gasket_dev);
 286                 return 0;
 287         }
 288
 289         sysfs_type = (enum interrupt_sysfs_attribute_type)
 290                 gasket_attr->data.attr_type;
 291         interrupt_data = gasket_dev->interrupt_data;
 292         switch (sysfs_type) {
 293         case ATTR_INTERRUPT_COUNTS:
 294                 for (i = 0; i < interrupt_data->num_interrupts; ++i) {
 295                         written =
 296                                 scnprintf(buf, PAGE_SIZE - total_written,
 297                                           "0x%02x: %ld\n", i,
 298                                           interrupt_data->interrupt_counts[i]);
 299                         total_written += written;
 300                         buf += written;
 301                 }
 302                 ret = total_written;
 303                 break;
 304         default:
 305                 dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n",
 306                         attr->attr.name);
 307                 ret = 0;
 308                 break;
 309         }
 310
 311         gasket_sysfs_put_attr(device, gasket_attr);
 312         gasket_sysfs_put_device_data(device, gasket_dev);
 313         return ret;
 314 }
 315
 316 static struct gasket_sysfs_attribute interrupt_sysfs_attrs[] = {
 317         GASKET_SYSFS_RO(interrupt_counts, interrupt_sysfs_show,
 318                         ATTR_INTERRUPT_COUNTS),
 319         GASKET_END_OF_ATTR_ARRAY,
 320 };
 321
 322 int gasket_interrupt_init(struct gasket_dev *gasket_dev, const char *name,
 323                           int type,
 324                           const struct gasket_interrupt_desc *interrupts,
 325                           int num_interrupts, int pack_width, int bar_index,
 326                           const struct gasket_wire_interrupt_offsets *wire_int_offsets)
 327 {
 328         int ret;
 329         struct gasket_interrupt_data *interrupt_data;
 330
 331         interrupt_data = kzalloc(sizeof(struct gasket_interrupt_data),
 332                                  GFP_KERNEL);
 333         if (!interrupt_data)
 334                 return -ENOMEM;
 335         gasket_dev->interrupt_data = interrupt_data;
 336         interrupt_data->name = name;
 337         interrupt_data->type = type;
 338         interrupt_data->pci_dev = gasket_dev->pci_dev;
 339         interrupt_data->num_interrupts = num_interrupts;
 340         interrupt_data->interrupts = interrupts;
 341         interrupt_data->interrupt_bar_index = bar_index;
 342         interrupt_data->pack_width = pack_width;
 343         interrupt_data->num_configured = 0;
 344         interrupt_data->wire_interrupt_offsets = wire_int_offsets;
 345
 346         /* Allocate all dynamic structures. */
 347         interrupt_data->msix_entries = kcalloc(num_interrupts,
 348                                                sizeof(struct msix_entry),
 349                                                GFP_KERNEL);
 350         if (!interrupt_data->msix_entries) {
 351                 kfree(interrupt_data);
 352                 return -ENOMEM;
 353         }
 354
 355         interrupt_data->eventfd_ctxs = kcalloc(num_interrupts,
 356                                                sizeof(struct eventfd_ctx *),
 357                                                GFP_KERNEL);
 358         if (!interrupt_data->eventfd_ctxs) {
 359                 kfree(interrupt_data->msix_entries);
 360                 kfree(interrupt_data);
 361                 return -ENOMEM;
 362         }
 363
 364         interrupt_data->interrupt_counts = kcalloc(num_interrupts,
 365                                                    sizeof(ulong),
 366                                                    GFP_KERNEL);
 367         if (!interrupt_data->interrupt_counts) {
 368                 kfree(interrupt_data->eventfd_ctxs);
 369                 kfree(interrupt_data->msix_entries);
 370                 kfree(interrupt_data);
 371                 return -ENOMEM;
 372         }
 373
 374         switch (interrupt_data->type) {
 375         case PCI_MSIX:
 376                 ret = gasket_interrupt_msix_init(interrupt_data);
 377                 if (ret)
 378                         break;
 379                 force_msix_interrupt_unmasking(gasket_dev);
 380                 break;
 381
 382         case PCI_MSI:
 383         case PLATFORM_WIRE:
 384         default:
 385                 dev_err(gasket_dev->dev,
 386                         "Cannot handle unsupported interrupt type %d\n",
 387                         interrupt_data->type);
 388                 ret = -EINVAL;
 389         }
 390
 391         if (ret) {
 392                 /* Failing to setup interrupts will cause the device to report
 393                  * GASKET_STATUS_LAMED. But it is not fatal.
 394                  */
 395                 dev_warn(gasket_dev->dev,
 396                          "Couldn't initialize interrupts: %d\n", ret);
 397                 return 0;
 398         }
 399
 400         gasket_interrupt_setup(gasket_dev);
 401         gasket_sysfs_create_entries(gasket_dev->dev_info.device,
 402                                     interrupt_sysfs_attrs);
 403
 404         return 0;
 405 }
 406
 407 static void
 408 gasket_interrupt_msix_cleanup(struct gasket_interrupt_data *interrupt_data)
 409 {
 410         int i;
 411
 412         for (i = 0; i < interrupt_data->num_configured; i++)
 413                 free_irq(interrupt_data->msix_entries[i].vector,
 414                          interrupt_data);
 415         interrupt_data->num_configured = 0;
 416
 417         if (interrupt_data->msix_configured)
 418                 pci_disable_msix(interrupt_data->pci_dev);
 419         interrupt_data->msix_configured = 0;
 420 }
 421
 422 int gasket_interrupt_reinit(struct gasket_dev *gasket_dev)
 423 {
 424         int ret;
 425
 426         if (!gasket_dev->interrupt_data) {
 427                 dev_dbg(gasket_dev->dev,
 428                         "Attempted to reinit uninitialized interrupt data\n");
 429                 return -EINVAL;
 430         }
 431
 432         switch (gasket_dev->interrupt_data->type) {
 433         case PCI_MSIX:
 434                 gasket_interrupt_msix_cleanup(gasket_dev->interrupt_data);
 435                 ret = gasket_interrupt_msix_init(gasket_dev->interrupt_data);
 436                 if (ret)
 437                         break;
 438                 force_msix_interrupt_unmasking(gasket_dev);
 439                 break;
 440
 441         case PCI_MSI:
 442         case PLATFORM_WIRE:
 443         default:
 444                 dev_dbg(gasket_dev->dev,
 445                         "Cannot handle unsupported interrupt type %d\n",
 446                         gasket_dev->interrupt_data->type);
 447                 ret = -EINVAL;
 448         }
 449
 450         if (ret) {
 451                 /* Failing to setup MSIx will cause the device
 452                  * to report GASKET_STATUS_LAMED, but is not fatal.
 453                  */
 454                 dev_warn(gasket_dev->dev, "Couldn't init msix: %d\n", ret);
 455                 return 0;
 456         }
 457
 458         gasket_interrupt_setup(gasket_dev);
 459
 460         return 0;
 461 }
 462
 463 /* See gasket_interrupt.h for description. */
 464 int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev)
 465 {
 466         dev_dbg(gasket_dev->dev, "Clearing interrupt counts\n");
 467         memset(gasket_dev->interrupt_data->interrupt_counts, 0,
 468                gasket_dev->interrupt_data->num_interrupts *
 469                         sizeof(*gasket_dev->interrupt_data->interrupt_counts));
 470         return 0;
 471 }
 472
 473 /* See gasket_interrupt.h for description. */
 474 void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev)
 475 {
 476         struct gasket_interrupt_data *interrupt_data =
 477                 gasket_dev->interrupt_data;
 478         /*
 479          * It is possible to get an error code from gasket_interrupt_init
 480          * before interrupt_data has been allocated, so check it.
 481          */
 482         if (!interrupt_data)
 483                 return;
 484
 485         switch (interrupt_data->type) {
 486         case PCI_MSIX:
 487                 gasket_interrupt_msix_cleanup(interrupt_data);
 488                 break;
 489
 490         case PCI_MSI:
 491         case PLATFORM_WIRE:
 492         default:
 493                 dev_dbg(gasket_dev->dev,
 494                         "Cannot handle unsupported interrupt type %d\n",
 495                         interrupt_data->type);
 496         }
 497
 498         kfree(interrupt_data->interrupt_counts);
 499         kfree(interrupt_data->eventfd_ctxs);
 500         kfree(interrupt_data->msix_entries);
 501         kfree(interrupt_data);
 502         gasket_dev->interrupt_data = NULL;
 503 }
 504
 505 int gasket_interrupt_system_status(struct gasket_dev *gasket_dev)
 506 {
 507         if (!gasket_dev->interrupt_data) {
 508                 dev_dbg(gasket_dev->dev, "Interrupt data is null\n");
 509                 return GASKET_STATUS_DEAD;
 510         }
 511
 512         if (!gasket_dev->interrupt_data->msix_configured) {
 513                 dev_dbg(gasket_dev->dev, "Interrupt not initialized\n");
 514                 return GASKET_STATUS_LAMED;
 515         }
 516
 517         if (gasket_dev->interrupt_data->num_configured !=
 518                 gasket_dev->interrupt_data->num_interrupts) {
 519                 dev_dbg(gasket_dev->dev,
 520                         "Not all interrupts were configured\n");
 521                 return GASKET_STATUS_LAMED;
 522         }
 523
 524         return GASKET_STATUS_ALIVE;
 525 }
 526
 527 int gasket_interrupt_set_eventfd(struct gasket_interrupt_data *interrupt_data,
 528                                  int interrupt, int event_fd)
 529 {
 530         struct eventfd_ctx *ctx = eventfd_ctx_fdget(event_fd);
 531
 532         if (IS_ERR(ctx))
 533                 return PTR_ERR(ctx);
 534
 535         if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts)
 536                 return -EINVAL;
 537
 538         interrupt_data->eventfd_ctxs[interrupt] = ctx;
 539         return 0;
 540 }
 541
 542 int gasket_interrupt_clear_eventfd(struct gasket_interrupt_data *interrupt_data,
 543                                    int interrupt)
 544 {
 545         if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts)
 546                 return -EINVAL;
 547
 548         interrupt_data->eventfd_ctxs[interrupt] = NULL;
 549         return 0;
 550 }