drivers/staging/gasket/gasket_interrupt.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (C) 2018 Google, Inc. */
   3
   4 #include "gasket_interrupt.h"
   5
   6 #include "gasket_constants.h"
   7 #include "gasket_core.h"
   8 #include "gasket_sysfs.h"
   9 #include <linux/device.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/printk.h>
  12 #ifdef GASKET_KERNEL_TRACE_SUPPORT
  13 #define CREATE_TRACE_POINTS
  14 #include <trace/events/gasket_interrupt.h>
  15 #else
  16 #define trace_gasket_interrupt_event(x, ...)
  17 #endif
  18 /* Retry attempts if the requested number of interrupts aren't available. */
  19 #define MSIX_RETRY_COUNT 3
  20
  21 /* Instance interrupt management data. */
  22 struct gasket_interrupt_data {
  23         /* The name associated with this interrupt data. */
  24         const char *name;
  25
  26         /* Interrupt type. See gasket_interrupt_type in gasket_core.h */
  27         int type;
  28
  29         /* The PCI device [if any] associated with the owning device. */
  30         struct pci_dev *pci_dev;
  31
  32         /* Set to 1 if MSI-X has successfully been configred, 0 otherwise. */
  33         int msix_configured;
  34
  35         /* The number of interrupts requested by the owning device. */
  36         int num_interrupts;
  37
  38         /* A pointer to the interrupt descriptor struct for this device. */
  39         const struct gasket_interrupt_desc *interrupts;
  40
  41         /* The index of the bar into which interrupts should be mapped. */
  42         int interrupt_bar_index;
  43
  44         /* The width of a single interrupt in a packed interrupt register. */
  45         int pack_width;
  46
  47         /*
  48          * Design-wise, these elements should be bundled together, but
  49          * pci_enable_msix's interface requires that they be managed
  50          * individually (requires array of struct msix_entry).
  51          */
  52
  53         /* The number of successfully configured interrupts. */
  54         int num_configured;
  55
  56         /* The MSI-X data for each requested/configured interrupt. */
  57         struct msix_entry *msix_entries;
  58
  59         /* The eventfd "callback" data for each interrupt. */
  60         struct eventfd_ctx **eventfd_ctxs;
  61
  62         /* The number of times each interrupt has been called. */
  63         ulong *interrupt_counts;
  64
  65         /* Linux IRQ number. */
  66         int irq;
  67 };
  68
  69 /* Structures to display interrupt counts in sysfs. */
  70 enum interrupt_sysfs_attribute_type {
  71         ATTR_INTERRUPT_COUNTS,
  72 };
  73
  74 /* Set up device registers for interrupt handling. */
  75 static void gasket_interrupt_setup(struct gasket_dev *gasket_dev)
  76 {
  77         int i;
  78         int pack_shift;
  79         ulong mask;
  80         ulong value;
  81         struct gasket_interrupt_data *interrupt_data =
  82                 gasket_dev->interrupt_data;
  83
  84         if (!interrupt_data) {
  85                 dev_dbg(gasket_dev->dev, "Interrupt data is not initialized\n");
  86                 return;
  87         }
  88
  89         dev_dbg(gasket_dev->dev, "Running interrupt setup\n");
  90
  91         /* Setup the MSIX table. */
  92
  93         for (i = 0; i < interrupt_data->num_interrupts; i++) {
  94                 /*
  95                  * If the interrupt is not packed, we can write the index into
  96                  * the register directly. If not, we need to deal with a read-
  97                  * modify-write and shift based on the packing index.
  98                  */
  99                 dev_dbg(gasket_dev->dev,
 100                         "Setting up interrupt index %d with index 0x%llx and packing %d\n",
 101                         interrupt_data->interrupts[i].index,
 102                         interrupt_data->interrupts[i].reg,
 103                         interrupt_data->interrupts[i].packing);
 104                 if (interrupt_data->interrupts[i].packing == UNPACKED) {
 105                         value = interrupt_data->interrupts[i].index;
 106                 } else {
 107                         switch (interrupt_data->interrupts[i].packing) {
 108                         case PACK_0:
 109                                 pack_shift = 0;
 110                                 break;
 111                         case PACK_1:
 112                                 pack_shift = interrupt_data->pack_width;
 113                                 break;
 114                         case PACK_2:
 115                                 pack_shift = 2 * interrupt_data->pack_width;
 116                                 break;
 117                         case PACK_3:
 118                                 pack_shift = 3 * interrupt_data->pack_width;
 119                                 break;
 120                         default:
 121                                 dev_dbg(gasket_dev->dev,
 122                                         "Found interrupt description with unknown enum %d\n",
 123                                         interrupt_data->interrupts[i].packing);
 124                                 return;
 125                         }
 126
 127                         mask = ~(0xFFFF << pack_shift);
 128                         value = gasket_dev_read_64(gasket_dev,
 129                                                    interrupt_data->interrupt_bar_index,
 130                                                    interrupt_data->interrupts[i].reg);
 131                         value &= mask;
 132                         value |= interrupt_data->interrupts[i].index
 133                                  << pack_shift;
 134                 }
 135                 gasket_dev_write_64(gasket_dev, value,
 136                                     interrupt_data->interrupt_bar_index,
 137                                     interrupt_data->interrupts[i].reg);
 138         }
 139 }
 140
 141 static void
 142 gasket_handle_interrupt(struct gasket_interrupt_data *interrupt_data,
 143                         int interrupt_index)
 144 {
 145         struct eventfd_ctx *ctx;
 146
 147         trace_gasket_interrupt_event(interrupt_data->name, interrupt_index);
 148         ctx = interrupt_data->eventfd_ctxs[interrupt_index];
 149         if (ctx)
 150                 eventfd_signal(ctx, 1);
 151
 152         ++(interrupt_data->interrupt_counts[interrupt_index]);
 153 }
 154
 155 static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id)
 156 {
 157         struct gasket_interrupt_data *interrupt_data = dev_id;
 158         int interrupt = -1;
 159         int i;
 160
 161         /* If this linear lookup is a problem, we can maintain a map/hash. */
 162         for (i = 0; i < interrupt_data->num_interrupts; i++) {
 163                 if (interrupt_data->msix_entries[i].vector == irq) {
 164                         interrupt = interrupt_data->msix_entries[i].entry;
 165                         break;
 166                 }
 167         }
 168         if (interrupt == -1) {
 169                 pr_err("Received unknown irq %d\n", irq);
 170                 return IRQ_HANDLED;
 171         }
 172         gasket_handle_interrupt(interrupt_data, interrupt);
 173         return IRQ_HANDLED;
 174 }
 175
 176 static int
 177 gasket_interrupt_msix_init(struct gasket_interrupt_data *interrupt_data)
 178 {
 179         int ret = 1;
 180         int i;
 181
 182         interrupt_data->msix_entries =
 183                 kcalloc(interrupt_data->num_interrupts,
 184                         sizeof(*interrupt_data->msix_entries), GFP_KERNEL);
 185         if (!interrupt_data->msix_entries)
 186                 return -ENOMEM;
 187
 188         for (i = 0; i < interrupt_data->num_interrupts; i++) {
 189                 interrupt_data->msix_entries[i].entry = i;
 190                 interrupt_data->msix_entries[i].vector = 0;
 191                 interrupt_data->eventfd_ctxs[i] = NULL;
 192         }
 193
 194         /* Retry MSIX_RETRY_COUNT times if not enough IRQs are available. */
 195         for (i = 0; i < MSIX_RETRY_COUNT && ret > 0; i++)
 196                 ret = pci_enable_msix_exact(interrupt_data->pci_dev,
 197                                             interrupt_data->msix_entries,
 198                                             interrupt_data->num_interrupts);
 199
 200         if (ret)
 201                 return ret > 0 ? -EBUSY : ret;
 202         interrupt_data->msix_configured = 1;
 203
 204         for (i = 0; i < interrupt_data->num_interrupts; i++) {
 205                 ret = request_irq(interrupt_data->msix_entries[i].vector,
 206                                   gasket_msix_interrupt_handler, 0,
 207                                   interrupt_data->name, interrupt_data);
 208
 209                 if (ret) {
 210                         dev_err(&interrupt_data->pci_dev->dev,
 211                                 "Cannot get IRQ for interrupt %d, vector %d; "
 212                                 "%d\n",
 213                                 i, interrupt_data->msix_entries[i].vector, ret);
 214                         return ret;
 215                 }
 216
 217                 interrupt_data->num_configured++;
 218         }
 219
 220         return 0;
 221 }
 222
 223 /*
 224  * On QCM DragonBoard, we exit gasket_interrupt_msix_init() and kernel interrupt
 225  * setup code with MSIX vectors masked. This is wrong because nothing else in
 226  * the driver will normally touch the MSIX vectors.
 227  *
 228  * As a temporary hack, force unmasking there.
 229  *
 230  * TODO: Figure out why QCM kernel doesn't unmask the MSIX vectors, after
 231  * gasket_interrupt_msix_init(), and remove this code.
 232  */
 233 static void force_msix_interrupt_unmasking(struct gasket_dev *gasket_dev)
 234 {
 235         int i;
 236 #define MSIX_VECTOR_SIZE 16
 237 #define MSIX_MASK_BIT_OFFSET 12
 238 #define APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE 0x46800
 239         for (i = 0; i < gasket_dev->interrupt_data->num_configured; i++) {
 240                 /* Check if the MSIX vector is unmasked */
 241                 ulong location = APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE +
 242                                  MSIX_MASK_BIT_OFFSET + i * MSIX_VECTOR_SIZE;
 243                 u32 mask =
 244                         gasket_dev_read_32(gasket_dev,
 245                                            gasket_dev->interrupt_data->interrupt_bar_index,
 246                                            location);
 247                 if (!(mask & 1))
 248                         continue;
 249                 /* Unmask the msix vector (clear 32 bits) */
 250                 gasket_dev_write_32(gasket_dev, 0,
 251                                     gasket_dev->interrupt_data->interrupt_bar_index,
 252                                     location);
 253         }
 254 #undef MSIX_VECTOR_SIZE
 255 #undef MSIX_MASK_BIT_OFFSET
 256 #undef APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE
 257 }
 258
 259 static ssize_t interrupt_sysfs_show(struct device *device,
 260                                     struct device_attribute *attr, char *buf)
 261 {
 262         int i, ret;
 263         ssize_t written = 0, total_written = 0;
 264         struct gasket_interrupt_data *interrupt_data;
 265         struct gasket_dev *gasket_dev;
 266         struct gasket_sysfs_attribute *gasket_attr;
 267         enum interrupt_sysfs_attribute_type sysfs_type;
 268
 269         gasket_dev = gasket_sysfs_get_device_data(device);
 270         if (!gasket_dev) {
 271                 dev_dbg(device, "No sysfs mapping found for device\n");
 272                 return 0;
 273         }
 274
 275         gasket_attr = gasket_sysfs_get_attr(device, attr);
 276         if (!gasket_attr) {
 277                 dev_dbg(device, "No sysfs attr data found for device\n");
 278                 gasket_sysfs_put_device_data(device, gasket_dev);
 279                 return 0;
 280         }
 281
 282         sysfs_type = (enum interrupt_sysfs_attribute_type)
 283                 gasket_attr->data.attr_type;
 284         interrupt_data = gasket_dev->interrupt_data;
 285         switch (sysfs_type) {
 286         case ATTR_INTERRUPT_COUNTS:
 287                 for (i = 0; i < interrupt_data->num_interrupts; ++i) {
 288                         written =
 289                                 scnprintf(buf, PAGE_SIZE - total_written,
 290                                           "0x%02x: %ld\n", i,
 291                                           interrupt_data->interrupt_counts[i]);
 292                         total_written += written;
 293                         buf += written;
 294                 }
 295                 ret = total_written;
 296                 break;
 297         default:
 298                 dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n",
 299                         attr->attr.name);
 300                 ret = 0;
 301                 break;
 302         }
 303
 304         gasket_sysfs_put_attr(device, gasket_attr);
 305         gasket_sysfs_put_device_data(device, gasket_dev);
 306         return ret;
 307 }
 308
 309 static struct gasket_sysfs_attribute interrupt_sysfs_attrs[] = {
 310         GASKET_SYSFS_RO(interrupt_counts, interrupt_sysfs_show,
 311                         ATTR_INTERRUPT_COUNTS),
 312         GASKET_END_OF_ATTR_ARRAY,
 313 };
 314
 315 int gasket_interrupt_init(struct gasket_dev *gasket_dev)
 316 {
 317         int ret;
 318         struct gasket_interrupt_data *interrupt_data;
 319         const struct gasket_driver_desc *driver_desc =
 320                 gasket_get_driver_desc(gasket_dev);
 321
 322         interrupt_data = kzalloc(sizeof(*interrupt_data), GFP_KERNEL);
 323         if (!interrupt_data)
 324                 return -ENOMEM;
 325         gasket_dev->interrupt_data = interrupt_data;
 326         interrupt_data->name = driver_desc->name;
 327         interrupt_data->type = driver_desc->interrupt_type;
 328         interrupt_data->pci_dev = gasket_dev->pci_dev;
 329         interrupt_data->num_interrupts = driver_desc->num_interrupts;
 330         interrupt_data->interrupts = driver_desc->interrupts;
 331         interrupt_data->interrupt_bar_index = driver_desc->interrupt_bar_index;
 332         interrupt_data->pack_width = driver_desc->interrupt_pack_width;
 333         interrupt_data->num_configured = 0;
 334
 335         interrupt_data->eventfd_ctxs =
 336                 kcalloc(driver_desc->num_interrupts,
 337                         sizeof(*interrupt_data->eventfd_ctxs), GFP_KERNEL);
 338         if (!interrupt_data->eventfd_ctxs) {
 339                 kfree(interrupt_data);
 340                 return -ENOMEM;
 341         }
 342
 343         interrupt_data->interrupt_counts =
 344                 kcalloc(driver_desc->num_interrupts,
 345                         sizeof(*interrupt_data->interrupt_counts), GFP_KERNEL);
 346         if (!interrupt_data->interrupt_counts) {
 347                 kfree(interrupt_data->eventfd_ctxs);
 348                 kfree(interrupt_data);
 349                 return -ENOMEM;
 350         }
 351
 352         switch (interrupt_data->type) {
 353         case PCI_MSIX:
 354                 ret = gasket_interrupt_msix_init(interrupt_data);
 355                 if (ret)
 356                         break;
 357                 force_msix_interrupt_unmasking(gasket_dev);
 358                 break;
 359
 360         default:
 361                 ret = -EINVAL;
 362         }
 363
 364         if (ret) {
 365                 /* Failing to setup interrupts will cause the device to report
 366                  * GASKET_STATUS_LAMED. But it is not fatal.
 367                  */
 368                 dev_warn(gasket_dev->dev,
 369                          "Couldn't initialize interrupts: %d\n", ret);
 370                 return 0;
 371         }
 372
 373         gasket_interrupt_setup(gasket_dev);
 374         gasket_sysfs_create_entries(gasket_dev->dev_info.device,
 375                                     interrupt_sysfs_attrs);
 376
 377         return 0;
 378 }
 379
 380 static void
 381 gasket_interrupt_msix_cleanup(struct gasket_interrupt_data *interrupt_data)
 382 {
 383         int i;
 384
 385         for (i = 0; i < interrupt_data->num_configured; i++)
 386                 free_irq(interrupt_data->msix_entries[i].vector,
 387                          interrupt_data);
 388         interrupt_data->num_configured = 0;
 389
 390         if (interrupt_data->msix_configured)
 391                 pci_disable_msix(interrupt_data->pci_dev);
 392         interrupt_data->msix_configured = 0;
 393         kfree(interrupt_data->msix_entries);
 394 }
 395
 396 int gasket_interrupt_reinit(struct gasket_dev *gasket_dev)
 397 {
 398         int ret;
 399
 400         if (!gasket_dev->interrupt_data) {
 401                 dev_dbg(gasket_dev->dev,
 402                         "Attempted to reinit uninitialized interrupt data\n");
 403                 return -EINVAL;
 404         }
 405
 406         switch (gasket_dev->interrupt_data->type) {
 407         case PCI_MSIX:
 408                 gasket_interrupt_msix_cleanup(gasket_dev->interrupt_data);
 409                 ret = gasket_interrupt_msix_init(gasket_dev->interrupt_data);
 410                 if (ret)
 411                         break;
 412                 force_msix_interrupt_unmasking(gasket_dev);
 413                 break;
 414
 415         default:
 416                 ret = -EINVAL;
 417         }
 418
 419         if (ret) {
 420                 /* Failing to setup interrupts will cause the device
 421                  * to report GASKET_STATUS_LAMED, but is not fatal.
 422                  */
 423                 dev_warn(gasket_dev->dev, "Couldn't reinit interrupts: %d\n",
 424                          ret);
 425                 return 0;
 426         }
 427
 428         gasket_interrupt_setup(gasket_dev);
 429
 430         return 0;
 431 }
 432
 433 /* See gasket_interrupt.h for description. */
 434 int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev)
 435 {
 436         dev_dbg(gasket_dev->dev, "Clearing interrupt counts\n");
 437         memset(gasket_dev->interrupt_data->interrupt_counts, 0,
 438                gasket_dev->interrupt_data->num_interrupts *
 439                         sizeof(*gasket_dev->interrupt_data->interrupt_counts));
 440         return 0;
 441 }
 442
 443 /* See gasket_interrupt.h for description. */
 444 void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev)
 445 {
 446         struct gasket_interrupt_data *interrupt_data =
 447                 gasket_dev->interrupt_data;
 448         /*
 449          * It is possible to get an error code from gasket_interrupt_init
 450          * before interrupt_data has been allocated, so check it.
 451          */
 452         if (!interrupt_data)
 453                 return;
 454
 455         switch (interrupt_data->type) {
 456         case PCI_MSIX:
 457                 gasket_interrupt_msix_cleanup(interrupt_data);
 458                 break;
 459
 460         default:
 461                 break;
 462         }
 463
 464         kfree(interrupt_data->interrupt_counts);
 465         kfree(interrupt_data->eventfd_ctxs);
 466         kfree(interrupt_data);
 467         gasket_dev->interrupt_data = NULL;
 468 }
 469
 470 int gasket_interrupt_system_status(struct gasket_dev *gasket_dev)
 471 {
 472         if (!gasket_dev->interrupt_data) {
 473                 dev_dbg(gasket_dev->dev, "Interrupt data is null\n");
 474                 return GASKET_STATUS_DEAD;
 475         }
 476
 477         if (gasket_dev->interrupt_data->num_configured !=
 478                 gasket_dev->interrupt_data->num_interrupts) {
 479                 dev_dbg(gasket_dev->dev,
 480                         "Not all interrupts were configured\n");
 481                 return GASKET_STATUS_LAMED;
 482         }
 483
 484         return GASKET_STATUS_ALIVE;
 485 }
 486
 487 int gasket_interrupt_set_eventfd(struct gasket_interrupt_data *interrupt_data,
 488                                  int interrupt, int event_fd)
 489 {
 490         struct eventfd_ctx *ctx = eventfd_ctx_fdget(event_fd);
 491
 492         if (IS_ERR(ctx))
 493                 return PTR_ERR(ctx);
 494
 495         if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts)
 496                 return -EINVAL;
 497
 498         interrupt_data->eventfd_ctxs[interrupt] = ctx;
 499         return 0;
 500 }
 501
 502 int gasket_interrupt_clear_eventfd(struct gasket_interrupt_data *interrupt_data,
 503                                    int interrupt)
 504 {
 505         if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts)
 506                 return -EINVAL;
 507
 508         interrupt_data->eventfd_ctxs[interrupt] = NULL;
 509         return 0;
 510 }