drivers/iommu/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #define pr_fmt(fmt)     "DMAR: " fmt
  30
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/iova.h>
  34 #include <linux/intel-iommu.h>
  35 #include <linux/timer.h>
  36 #include <linux/irq.h>
  37 #include <linux/interrupt.h>
  38 #include <linux/tboot.h>
  39 #include <linux/dmi.h>
  40 #include <linux/slab.h>
  41 #include <linux/iommu.h>
  42 #include <asm/irq_remapping.h>
  43 #include <asm/iommu_table.h>
  44
  45 #include "irq_remapping.h"
  46
  47 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
  48 struct dmar_res_callback {
  49         dmar_res_handler_t      cb[ACPI_DMAR_TYPE_RESERVED];
  50         void                    *arg[ACPI_DMAR_TYPE_RESERVED];
  51         bool                    ignore_unhandled;
  52         bool                    print_entry;
  53 };
  54
  55 /*
  56  * Assumptions:
  57  * 1) The hotplug framework guarentees that DMAR unit will be hot-added
  58  *    before IO devices managed by that unit.
  59  * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
  60  *    after IO devices managed by that unit.
  61  * 3) Hotplug events are rare.
  62  *
  63  * Locking rules for DMA and interrupt remapping related global data structures:
  64  * 1) Use dmar_global_lock in process context
  65  * 2) Use RCU in interrupt context
  66  */
  67 DECLARE_RWSEM(dmar_global_lock);
  68 LIST_HEAD(dmar_drhd_units);
  69
  70 struct acpi_table_header * __initdata dmar_tbl;
  71 static acpi_size dmar_tbl_size;
  72 static int dmar_dev_scope_status = 1;
  73 static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
  74
  75 static int alloc_iommu(struct dmar_drhd_unit *drhd);
  76 static void free_iommu(struct intel_iommu *iommu);
  77
  78 static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  79 {
  80         /*
  81          * add INCLUDE_ALL at the tail, so scan the list will find it at
  82          * the very end.
  83          */
  84         if (drhd->include_all)
  85                 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
  86         else
  87                 list_add_rcu(&drhd->list, &dmar_drhd_units);
  88 }
  89
  90 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
  91 {
  92         struct acpi_dmar_device_scope *scope;
  93
  94         *cnt = 0;
  95         while (start < end) {
  96                 scope = start;
  97                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
  98                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
  99                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 100                         (*cnt)++;
 101                 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
 102                         scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
 103                         pr_warn("Unsupported device scope\n");
 104                 }
 105                 start += scope->length;
 106         }
 107         if (*cnt == 0)
 108                 return NULL;
 109
 110         return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
 111 }
 112
 113 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
 114 {
 115         int i;
 116         struct device *tmp_dev;
 117
 118         if (*devices && *cnt) {
 119                 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
 120                         put_device(tmp_dev);
 121                 kfree(*devices);
 122         }
 123
 124         *devices = NULL;
 125         *cnt = 0;
 126 }
 127
 128 /* Optimize out kzalloc()/kfree() for normal cases */
 129 static char dmar_pci_notify_info_buf[64];
 130
 131 static struct dmar_pci_notify_info *
 132 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
 133 {
 134         int level = 0;
 135         size_t size;
 136         struct pci_dev *tmp;
 137         struct dmar_pci_notify_info *info;
 138
 139         BUG_ON(dev->is_virtfn);
 140
 141         /* Only generate path[] for device addition event */
 142         if (event == BUS_NOTIFY_ADD_DEVICE)
 143                 for (tmp = dev; tmp; tmp = tmp->bus->self)
 144                         level++;
 145
 146         size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
 147         if (size <= sizeof(dmar_pci_notify_info_buf)) {
 148                 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
 149         } else {
 150                 info = kzalloc(size, GFP_KERNEL);
 151                 if (!info) {
 152                         pr_warn("Out of memory when allocating notify_info "
 153                                 "for %s.\n", pci_name(dev));
 154                         if (dmar_dev_scope_status == 0)
 155                                 dmar_dev_scope_status = -ENOMEM;
 156                         return NULL;
 157                 }
 158         }
 159
 160         info->event = event;
 161         info->dev = dev;
 162         info->seg = pci_domain_nr(dev->bus);
 163         info->level = level;
 164         if (event == BUS_NOTIFY_ADD_DEVICE) {
 165                 for (tmp = dev; tmp; tmp = tmp->bus->self) {
 166                         level--;
 167                         info->path[level].bus = tmp->bus->number;
 168                         info->path[level].device = PCI_SLOT(tmp->devfn);
 169                         info->path[level].function = PCI_FUNC(tmp->devfn);
 170                         if (pci_is_root_bus(tmp->bus))
 171                                 info->bus = tmp->bus->number;
 172                 }
 173         }
 174
 175         return info;
 176 }
 177
 178 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
 179 {
 180         if ((void *)info != dmar_pci_notify_info_buf)
 181                 kfree(info);
 182 }
 183
 184 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
 185                                 struct acpi_dmar_pci_path *path, int count)
 186 {
 187         int i;
 188
 189         if (info->bus != bus)
 190                 goto fallback;
 191         if (info->level != count)
 192                 goto fallback;
 193
 194         for (i = 0; i < count; i++) {
 195                 if (path[i].device != info->path[i].device ||
 196                     path[i].function != info->path[i].function)
 197                         goto fallback;
 198         }
 199
 200         return true;
 201
 202 fallback:
 203
 204         if (count != 1)
 205                 return false;
 206
 207         i = info->level - 1;
 208         if (bus              == info->path[i].bus &&
 209             path[0].device   == info->path[i].device &&
 210             path[0].function == info->path[i].function) {
 211                 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
 212                         bus, path[0].device, path[0].function);
 213                 return true;
 214         }
 215
 216         return false;
 217 }
 218
 219 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
 220 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
 221                           void *start, void*end, u16 segment,
 222                           struct dmar_dev_scope *devices,
 223                           int devices_cnt)
 224 {
 225         int i, level;
 226         struct device *tmp, *dev = &info->dev->dev;
 227         struct acpi_dmar_device_scope *scope;
 228         struct acpi_dmar_pci_path *path;
 229
 230         if (segment != info->seg)
 231                 return 0;
 232
 233         for (; start < end; start += scope->length) {
 234                 scope = start;
 235                 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 236                     scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 237                         continue;
 238
 239                 path = (struct acpi_dmar_pci_path *)(scope + 1);
 240                 level = (scope->length - sizeof(*scope)) / sizeof(*path);
 241                 if (!dmar_match_pci_path(info, scope->bus, path, level))
 242                         continue;
 243
 244                 /*
 245                  * We expect devices with endpoint scope to have normal PCI
 246                  * headers, and devices with bridge scope to have bridge PCI
 247                  * headers.  However PCI NTB devices may be listed in the
 248                  * DMAR table with bridge scope, even though they have a
 249                  * normal PCI header.  NTB devices are identified by class
 250                  * "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch
 251                  * for this special case.
 252                  */
 253                 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 254                      info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) ||
 255                     (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
 256                      (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
 257                       info->dev->class >> 8 != PCI_CLASS_BRIDGE_OTHER))) {
 258                         pr_warn("Device scope type does not match for %s\n",
 259                                 pci_name(info->dev));
 260                         return -EINVAL;
 261                 }
 262
 263                 for_each_dev_scope(devices, devices_cnt, i, tmp)
 264                         if (tmp == NULL) {
 265                                 devices[i].bus = info->dev->bus->number;
 266                                 devices[i].devfn = info->dev->devfn;
 267                                 rcu_assign_pointer(devices[i].dev,
 268                                                    get_device(dev));
 269                                 return 1;
 270                         }
 271                 BUG_ON(i >= devices_cnt);
 272         }
 273
 274         return 0;
 275 }
 276
 277 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
 278                           struct dmar_dev_scope *devices, int count)
 279 {
 280         int index;
 281         struct device *tmp;
 282
 283         if (info->seg != segment)
 284                 return 0;
 285
 286         for_each_active_dev_scope(devices, count, index, tmp)
 287                 if (tmp == &info->dev->dev) {
 288                         RCU_INIT_POINTER(devices[index].dev, NULL);
 289                         synchronize_rcu();
 290                         put_device(tmp);
 291                         return 1;
 292                 }
 293
 294         return 0;
 295 }
 296
 297 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
 298 {
 299         int ret = 0;
 300         struct dmar_drhd_unit *dmaru;
 301         struct acpi_dmar_hardware_unit *drhd;
 302
 303         for_each_drhd_unit(dmaru) {
 304                 if (dmaru->include_all)
 305                         continue;
 306
 307                 drhd = container_of(dmaru->hdr,
 308                                     struct acpi_dmar_hardware_unit, header);
 309                 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
 310                                 ((void *)drhd) + drhd->header.length,
 311                                 dmaru->segment,
 312                                 dmaru->devices, dmaru->devices_cnt);
 313                 if (ret != 0)
 314                         break;
 315         }
 316         if (ret >= 0)
 317                 ret = dmar_iommu_notify_scope_dev(info);
 318         if (ret < 0 && dmar_dev_scope_status == 0)
 319                 dmar_dev_scope_status = ret;
 320
 321         return ret;
 322 }
 323
 324 static void  dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
 325 {
 326         struct dmar_drhd_unit *dmaru;
 327
 328         for_each_drhd_unit(dmaru)
 329                 if (dmar_remove_dev_scope(info, dmaru->segment,
 330                         dmaru->devices, dmaru->devices_cnt))
 331                         break;
 332         dmar_iommu_notify_scope_dev(info);
 333 }
 334
 335 static int dmar_pci_bus_notifier(struct notifier_block *nb,
 336                                  unsigned long action, void *data)
 337 {
 338         struct pci_dev *pdev = to_pci_dev(data);
 339         struct dmar_pci_notify_info *info;
 340
 341         /* Only care about add/remove events for physical functions.
 342          * For VFs we actually do the lookup based on the corresponding
 343          * PF in device_to_iommu() anyway. */
 344         if (pdev->is_virtfn)
 345                 return NOTIFY_DONE;
 346         if (action != BUS_NOTIFY_ADD_DEVICE &&
 347             action != BUS_NOTIFY_REMOVED_DEVICE)
 348                 return NOTIFY_DONE;
 349
 350         info = dmar_alloc_pci_notify_info(pdev, action);
 351         if (!info)
 352                 return NOTIFY_DONE;
 353
 354         down_write(&dmar_global_lock);
 355         if (action == BUS_NOTIFY_ADD_DEVICE)
 356                 dmar_pci_bus_add_dev(info);
 357         else if (action == BUS_NOTIFY_REMOVED_DEVICE)
 358                 dmar_pci_bus_del_dev(info);
 359         up_write(&dmar_global_lock);
 360
 361         dmar_free_pci_notify_info(info);
 362
 363         return NOTIFY_OK;
 364 }
 365
 366 static struct notifier_block dmar_pci_bus_nb = {
 367         .notifier_call = dmar_pci_bus_notifier,
 368         .priority = INT_MIN,
 369 };
 370
 371 static struct dmar_drhd_unit *
 372 dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
 373 {
 374         struct dmar_drhd_unit *dmaru;
 375
 376         list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list)
 377                 if (dmaru->segment == drhd->segment &&
 378                     dmaru->reg_base_addr == drhd->address)
 379                         return dmaru;
 380
 381         return NULL;
 382 }
 383
 384 /**
 385  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 386  * structure which uniquely represent one DMA remapping hardware unit
 387  * present in the platform
 388  */
 389 static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
 390 {
 391         struct acpi_dmar_hardware_unit *drhd;
 392         struct dmar_drhd_unit *dmaru;
 393         int ret = 0;
 394
 395         drhd = (struct acpi_dmar_hardware_unit *)header;
 396         dmaru = dmar_find_dmaru(drhd);
 397         if (dmaru)
 398                 goto out;
 399
 400         dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
 401         if (!dmaru)
 402                 return -ENOMEM;
 403
 404         /*
 405          * If header is allocated from slab by ACPI _DSM method, we need to
 406          * copy the content because the memory buffer will be freed on return.
 407          */
 408         dmaru->hdr = (void *)(dmaru + 1);
 409         memcpy(dmaru->hdr, header, header->length);
 410         dmaru->reg_base_addr = drhd->address;
 411         dmaru->segment = drhd->segment;
 412         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 413         dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
 414                                               ((void *)drhd) + drhd->header.length,
 415                                               &dmaru->devices_cnt);
 416         if (dmaru->devices_cnt && dmaru->devices == NULL) {
 417                 kfree(dmaru);
 418                 return -ENOMEM;
 419         }
 420
 421         ret = alloc_iommu(dmaru);
 422         if (ret) {
 423                 dmar_free_dev_scope(&dmaru->devices,
 424                                     &dmaru->devices_cnt);
 425                 kfree(dmaru);
 426                 return ret;
 427         }
 428         dmar_register_drhd_unit(dmaru);
 429
 430 out:
 431         if (arg)
 432                 (*(int *)arg)++;
 433
 434         return 0;
 435 }
 436
 437 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
 438 {
 439         if (dmaru->devices && dmaru->devices_cnt)
 440                 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
 441         if (dmaru->iommu)
 442                 free_iommu(dmaru->iommu);
 443         kfree(dmaru);
 444 }
 445
 446 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
 447                                       void *arg)
 448 {
 449         struct acpi_dmar_andd *andd = (void *)header;
 450
 451         /* Check for NUL termination within the designated length */
 452         if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
 453                 WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
 454                            "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
 455                            "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 456                            dmi_get_system_info(DMI_BIOS_VENDOR),
 457                            dmi_get_system_info(DMI_BIOS_VERSION),
 458                            dmi_get_system_info(DMI_PRODUCT_VERSION));
 459                 return -EINVAL;
 460         }
 461         pr_info("ANDD device: %x name: %s\n", andd->device_number,
 462                 andd->device_name);
 463
 464         return 0;
 465 }
 466
 467 #ifdef CONFIG_ACPI_NUMA
 468 static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
 469 {
 470         struct acpi_dmar_rhsa *rhsa;
 471         struct dmar_drhd_unit *drhd;
 472
 473         rhsa = (struct acpi_dmar_rhsa *)header;
 474         for_each_drhd_unit(drhd) {
 475                 if (drhd->reg_base_addr == rhsa->base_address) {
 476                         int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 477
 478                         if (!node_online(node))
 479                                 node = -1;
 480                         drhd->iommu->node = node;
 481                         return 0;
 482                 }
 483         }
 484         WARN_TAINT(
 485                 1, TAINT_FIRMWARE_WORKAROUND,
 486                 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 487                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 488                 drhd->reg_base_addr,
 489                 dmi_get_system_info(DMI_BIOS_VENDOR),
 490                 dmi_get_system_info(DMI_BIOS_VERSION),
 491                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 492
 493         return 0;
 494 }
 495 #else
 496 #define dmar_parse_one_rhsa             dmar_res_noop
 497 #endif
 498
 499 static void __init
 500 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 501 {
 502         struct acpi_dmar_hardware_unit *drhd;
 503         struct acpi_dmar_reserved_memory *rmrr;
 504         struct acpi_dmar_atsr *atsr;
 505         struct acpi_dmar_rhsa *rhsa;
 506
 507         switch (header->type) {
 508         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 509                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 510                                     header);
 511                 pr_info("DRHD base: %#016Lx flags: %#x\n",
 512                         (unsigned long long)drhd->address, drhd->flags);
 513                 break;
 514         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 515                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 516                                     header);
 517                 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 518                         (unsigned long long)rmrr->base_address,
 519                         (unsigned long long)rmrr->end_address);
 520                 break;
 521         case ACPI_DMAR_TYPE_ROOT_ATS:
 522                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 523                 pr_info("ATSR flags: %#x\n", atsr->flags);
 524                 break;
 525         case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
 526                 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 527                 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 528                        (unsigned long long)rhsa->base_address,
 529                        rhsa->proximity_domain);
 530                 break;
 531         case ACPI_DMAR_TYPE_NAMESPACE:
 532                 /* We don't print this here because we need to sanity-check
 533                    it first. So print it in dmar_parse_one_andd() instead. */
 534                 break;
 535         }
 536 }
 537
 538 /**
 539  * dmar_table_detect - checks to see if the platform supports DMAR devices
 540  */
 541 static int __init dmar_table_detect(void)
 542 {
 543         acpi_status status = AE_OK;
 544
 545         /* if we could find DMAR table, then there are DMAR devices */
 546         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 547                                 (struct acpi_table_header **)&dmar_tbl,
 548                                 &dmar_tbl_size);
 549
 550         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 551                 pr_warn("Unable to map DMAR\n");
 552                 status = AE_NOT_FOUND;
 553         }
 554
 555         return (ACPI_SUCCESS(status) ? 1 : 0);
 556 }
 557
 558 static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
 559                                        size_t len, struct dmar_res_callback *cb)
 560 {
 561         int ret = 0;
 562         struct acpi_dmar_header *iter, *next;
 563         struct acpi_dmar_header *end = ((void *)start) + len;
 564
 565         for (iter = start; iter < end && ret == 0; iter = next) {
 566                 next = (void *)iter + iter->length;
 567                 if (iter->length == 0) {
 568                         /* Avoid looping forever on bad ACPI tables */
 569                         pr_debug(FW_BUG "Invalid 0-length structure\n");
 570                         break;
 571                 } else if (next > end) {
 572                         /* Avoid passing table end */
 573                         pr_warn(FW_BUG "Record passes table end\n");
 574                         ret = -EINVAL;
 575                         break;
 576                 }
 577
 578                 if (cb->print_entry)
 579                         dmar_table_print_dmar_entry(iter);
 580
 581                 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
 582                         /* continue for forward compatibility */
 583                         pr_debug("Unknown DMAR structure type %d\n",
 584                                  iter->type);
 585                 } else if (cb->cb[iter->type]) {
 586                         ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
 587                 } else if (!cb->ignore_unhandled) {
 588                         pr_warn("No handler for DMAR structure type %d\n",
 589                                 iter->type);
 590                         ret = -EINVAL;
 591                 }
 592         }
 593
 594         return ret;
 595 }
 596
 597 static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
 598                                        struct dmar_res_callback *cb)
 599 {
 600         return dmar_walk_remapping_entries((void *)(dmar + 1),
 601                         dmar->header.length - sizeof(*dmar), cb);
 602 }
 603
 604 /**
 605  * parse_dmar_table - parses the DMA reporting table
 606  */
 607 static int __init
 608 parse_dmar_table(void)
 609 {
 610         struct acpi_table_dmar *dmar;
 611         int ret = 0;
 612         int drhd_count = 0;
 613         struct dmar_res_callback cb = {
 614                 .print_entry = true,
 615                 .ignore_unhandled = true,
 616                 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
 617                 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
 618                 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
 619                 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
 620                 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
 621                 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
 622         };
 623
 624         /*
 625          * Do it again, earlier dmar_tbl mapping could be mapped with
 626          * fixed map.
 627          */
 628         dmar_table_detect();
 629
 630         /*
 631          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 632          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 633          */
 634         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 635
 636         dmar = (struct acpi_table_dmar *)dmar_tbl;
 637         if (!dmar)
 638                 return -ENODEV;
 639
 640         if (dmar->width < PAGE_SHIFT - 1) {
 641                 pr_warn("Invalid DMAR haw\n");
 642                 return -EINVAL;
 643         }
 644
 645         pr_info("Host address width %d\n", dmar->width + 1);
 646         ret = dmar_walk_dmar_table(dmar, &cb);
 647         if (ret == 0 && drhd_count == 0)
 648                 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
 649
 650         return ret;
 651 }
 652
 653 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
 654                                  int cnt, struct pci_dev *dev)
 655 {
 656         int index;
 657         struct device *tmp;
 658
 659         while (dev) {
 660                 for_each_active_dev_scope(devices, cnt, index, tmp)
 661                         if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
 662                                 return 1;
 663
 664                 /* Check our parent */
 665                 dev = dev->bus->self;
 666         }
 667
 668         return 0;
 669 }
 670
 671 struct dmar_drhd_unit *
 672 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 673 {
 674         struct dmar_drhd_unit *dmaru;
 675         struct acpi_dmar_hardware_unit *drhd;
 676
 677         dev = pci_physfn(dev);
 678
 679         rcu_read_lock();
 680         for_each_drhd_unit(dmaru) {
 681                 drhd = container_of(dmaru->hdr,
 682                                     struct acpi_dmar_hardware_unit,
 683                                     header);
 684
 685                 if (dmaru->include_all &&
 686                     drhd->segment == pci_domain_nr(dev->bus))
 687                         goto out;
 688
 689                 if (dmar_pci_device_match(dmaru->devices,
 690                                           dmaru->devices_cnt, dev))
 691                         goto out;
 692         }
 693         dmaru = NULL;
 694 out:
 695         rcu_read_unlock();
 696
 697         return dmaru;
 698 }
 699
 700 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
 701                                               struct acpi_device *adev)
 702 {
 703         struct dmar_drhd_unit *dmaru;
 704         struct acpi_dmar_hardware_unit *drhd;
 705         struct acpi_dmar_device_scope *scope;
 706         struct device *tmp;
 707         int i;
 708         struct acpi_dmar_pci_path *path;
 709
 710         for_each_drhd_unit(dmaru) {
 711                 drhd = container_of(dmaru->hdr,
 712                                     struct acpi_dmar_hardware_unit,
 713                                     header);
 714
 715                 for (scope = (void *)(drhd + 1);
 716                      (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
 717                      scope = ((void *)scope) + scope->length) {
 718                         if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
 719                                 continue;
 720                         if (scope->enumeration_id != device_number)
 721                                 continue;
 722
 723                         path = (void *)(scope + 1);
 724                         pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
 725                                 dev_name(&adev->dev), dmaru->reg_base_addr,
 726                                 scope->bus, path->device, path->function);
 727                         for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
 728                                 if (tmp == NULL) {
 729                                         dmaru->devices[i].bus = scope->bus;
 730                                         dmaru->devices[i].devfn = PCI_DEVFN(path->device,
 731                                                                             path->function);
 732                                         rcu_assign_pointer(dmaru->devices[i].dev,
 733                                                            get_device(&adev->dev));
 734                                         return;
 735                                 }
 736                         BUG_ON(i >= dmaru->devices_cnt);
 737                 }
 738         }
 739         pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
 740                 device_number, dev_name(&adev->dev));
 741 }
 742
 743 static int __init dmar_acpi_dev_scope_init(void)
 744 {
 745         struct acpi_dmar_andd *andd;
 746
 747         if (dmar_tbl == NULL)
 748                 return -ENODEV;
 749
 750         for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
 751              ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
 752              andd = ((void *)andd) + andd->header.length) {
 753                 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
 754                         acpi_handle h;
 755                         struct acpi_device *adev;
 756
 757                         if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
 758                                                           andd->device_name,
 759                                                           &h))) {
 760                                 pr_err("Failed to find handle for ACPI object %s\n",
 761                                        andd->device_name);
 762                                 continue;
 763                         }
 764                         if (acpi_bus_get_device(h, &adev)) {
 765                                 pr_err("Failed to get device for ACPI object %s\n",
 766                                        andd->device_name);
 767                                 continue;
 768                         }
 769                         dmar_acpi_insert_dev_scope(andd->device_number, adev);
 770                 }
 771         }
 772         return 0;
 773 }
 774
 775 int __init dmar_dev_scope_init(void)
 776 {
 777         struct pci_dev *dev = NULL;
 778         struct dmar_pci_notify_info *info;
 779
 780         if (dmar_dev_scope_status != 1)
 781                 return dmar_dev_scope_status;
 782
 783         if (list_empty(&dmar_drhd_units)) {
 784                 dmar_dev_scope_status = -ENODEV;
 785         } else {
 786                 dmar_dev_scope_status = 0;
 787
 788                 dmar_acpi_dev_scope_init();
 789
 790                 for_each_pci_dev(dev) {
 791                         if (dev->is_virtfn)
 792                                 continue;
 793
 794                         info = dmar_alloc_pci_notify_info(dev,
 795                                         BUS_NOTIFY_ADD_DEVICE);
 796                         if (!info) {
 797                                 return dmar_dev_scope_status;
 798                         } else {
 799                                 dmar_pci_bus_add_dev(info);
 800                                 dmar_free_pci_notify_info(info);
 801                         }
 802                 }
 803
 804                 bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
 805         }
 806
 807         return dmar_dev_scope_status;
 808 }
 809
 810
 811 int __init dmar_table_init(void)
 812 {
 813         static int dmar_table_initialized;
 814         int ret;
 815
 816         if (dmar_table_initialized == 0) {
 817                 ret = parse_dmar_table();
 818                 if (ret < 0) {
 819                         if (ret != -ENODEV)
 820                                 pr_info("Parse DMAR table failure.\n");
 821                 } else  if (list_empty(&dmar_drhd_units)) {
 822                         pr_info("No DMAR devices found\n");
 823                         ret = -ENODEV;
 824                 }
 825
 826                 if (ret < 0)
 827                         dmar_table_initialized = ret;
 828                 else
 829                         dmar_table_initialized = 1;
 830         }
 831
 832         return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
 833 }
 834
 835 static void warn_invalid_dmar(u64 addr, const char *message)
 836 {
 837         WARN_TAINT_ONCE(
 838                 1, TAINT_FIRMWARE_WORKAROUND,
 839                 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 840                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 841                 addr, message,
 842                 dmi_get_system_info(DMI_BIOS_VENDOR),
 843                 dmi_get_system_info(DMI_BIOS_VERSION),
 844                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 845 }
 846
 847 static int __ref
 848 dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
 849 {
 850         struct acpi_dmar_hardware_unit *drhd;
 851         void __iomem *addr;
 852         u64 cap, ecap;
 853
 854         drhd = (void *)entry;
 855         if (!drhd->address) {
 856                 warn_invalid_dmar(0, "");
 857                 return -EINVAL;
 858         }
 859
 860         if (arg)
 861                 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
 862         else
 863                 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 864         if (!addr) {
 865                 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
 866                 return -EINVAL;
 867         }
 868
 869         cap = dmar_readq(addr + DMAR_CAP_REG);
 870         ecap = dmar_readq(addr + DMAR_ECAP_REG);
 871
 872         if (arg)
 873                 iounmap(addr);
 874         else
 875                 early_iounmap(addr, VTD_PAGE_SIZE);
 876
 877         if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 878                 warn_invalid_dmar(drhd->address, " returns all ones");
 879                 return -EINVAL;
 880         }
 881
 882         return 0;
 883 }
 884
 885 int __init detect_intel_iommu(void)
 886 {
 887         int ret;
 888         struct dmar_res_callback validate_drhd_cb = {
 889                 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
 890                 .ignore_unhandled = true,
 891         };
 892
 893         down_write(&dmar_global_lock);
 894         ret = dmar_table_detect();
 895         if (ret)
 896                 ret = !dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
 897                                             &validate_drhd_cb);
 898         if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 899                 iommu_detected = 1;
 900                 /* Make sure ACS will be enabled */
 901                 pci_request_acs();
 902         }
 903
 904 #ifdef CONFIG_X86
 905         if (ret)
 906                 x86_init.iommu.iommu_init = intel_iommu_init;
 907 #endif
 908
 909         early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
 910         dmar_tbl = NULL;
 911         up_write(&dmar_global_lock);
 912
 913         return ret ? 1 : -ENODEV;
 914 }
 915
 916
 917 static void unmap_iommu(struct intel_iommu *iommu)
 918 {
 919         iounmap(iommu->reg);
 920         release_mem_region(iommu->reg_phys, iommu->reg_size);
 921 }
 922
 923 /**
 924  * map_iommu: map the iommu's registers
 925  * @iommu: the iommu to map
 926  * @phys_addr: the physical address of the base resgister
 927  *
 928  * Memory map the iommu's registers.  Start w/ a single page, and
 929  * possibly expand if that turns out to be insufficent.
 930  */
 931 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 932 {
 933         int map_size, err=0;
 934
 935         iommu->reg_phys = phys_addr;
 936         iommu->reg_size = VTD_PAGE_SIZE;
 937
 938         if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 939                 pr_err("Can't reserve memory\n");
 940                 err = -EBUSY;
 941                 goto out;
 942         }
 943
 944         iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 945         if (!iommu->reg) {
 946                 pr_err("Can't map the region\n");
 947                 err = -ENOMEM;
 948                 goto release;
 949         }
 950
 951         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 952         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 953
 954         if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 955                 err = -EINVAL;
 956                 warn_invalid_dmar(phys_addr, " returns all ones");
 957                 goto unmap;
 958         }
 959
 960         /* the registers might be more than one page */
 961         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 962                          cap_max_fault_reg_offset(iommu->cap));
 963         map_size = VTD_PAGE_ALIGN(map_size);
 964         if (map_size > iommu->reg_size) {
 965                 iounmap(iommu->reg);
 966                 release_mem_region(iommu->reg_phys, iommu->reg_size);
 967                 iommu->reg_size = map_size;
 968                 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
 969                                         iommu->name)) {
 970                         pr_err("Can't reserve memory\n");
 971                         err = -EBUSY;
 972                         goto out;
 973                 }
 974                 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 975                 if (!iommu->reg) {
 976                         pr_err("Can't map the region\n");
 977                         err = -ENOMEM;
 978                         goto release;
 979                 }
 980         }
 981         err = 0;
 982         goto out;
 983
 984 unmap:
 985         iounmap(iommu->reg);
 986 release:
 987         release_mem_region(iommu->reg_phys, iommu->reg_size);
 988 out:
 989         return err;
 990 }
 991
 992 static int dmar_alloc_seq_id(struct intel_iommu *iommu)
 993 {
 994         iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
 995                                             DMAR_UNITS_SUPPORTED);
 996         if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
 997                 iommu->seq_id = -1;
 998         } else {
 999                 set_bit(iommu->seq_id, dmar_seq_ids);
1000                 sprintf(iommu->name, "dmar%d", iommu->seq_id);
1001         }
1002
1003         return iommu->seq_id;
1004 }
1005
1006 static void dmar_free_seq_id(struct intel_iommu *iommu)
1007 {
1008         if (iommu->seq_id >= 0) {
1009                 clear_bit(iommu->seq_id, dmar_seq_ids);
1010                 iommu->seq_id = -1;
1011         }
1012 }
1013
1014 static int alloc_iommu(struct dmar_drhd_unit *drhd)
1015 {
1016         struct intel_iommu *iommu;
1017         u32 ver, sts;
1018         int agaw = 0;
1019         int msagaw = 0;
1020         int err;
1021
1022         if (!drhd->reg_base_addr) {
1023                 warn_invalid_dmar(0, "");
1024                 return -EINVAL;
1025         }
1026
1027         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1028         if (!iommu)
1029                 return -ENOMEM;
1030
1031         if (dmar_alloc_seq_id(iommu) < 0) {
1032                 pr_err("Failed to allocate seq_id\n");
1033                 err = -ENOSPC;
1034                 goto error;
1035         }
1036
1037         err = map_iommu(iommu, drhd->reg_base_addr);
1038         if (err) {
1039                 pr_err("Failed to map %s\n", iommu->name);
1040                 goto error_free_seq_id;
1041         }
1042
1043         err = -EINVAL;
1044         agaw = iommu_calculate_agaw(iommu);
1045         if (agaw < 0) {
1046                 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1047                         iommu->seq_id);
1048                 goto err_unmap;
1049         }
1050         msagaw = iommu_calculate_max_sagaw(iommu);
1051         if (msagaw < 0) {
1052                 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1053                         iommu->seq_id);
1054                 goto err_unmap;
1055         }
1056         iommu->agaw = agaw;
1057         iommu->msagaw = msagaw;
1058         iommu->segment = drhd->segment;
1059
1060         iommu->node = -1;
1061
1062         ver = readl(iommu->reg + DMAR_VER_REG);
1063         pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1064                 iommu->name,
1065                 (unsigned long long)drhd->reg_base_addr,
1066                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1067                 (unsigned long long)iommu->cap,
1068                 (unsigned long long)iommu->ecap);
1069
1070         /* Reflect status in gcmd */
1071         sts = readl(iommu->reg + DMAR_GSTS_REG);
1072         if (sts & DMA_GSTS_IRES)
1073                 iommu->gcmd |= DMA_GCMD_IRE;
1074         if (sts & DMA_GSTS_TES)
1075                 iommu->gcmd |= DMA_GCMD_TE;
1076         if (sts & DMA_GSTS_QIES)
1077                 iommu->gcmd |= DMA_GCMD_QIE;
1078
1079         raw_spin_lock_init(&iommu->register_lock);
1080
1081         if (intel_iommu_enabled) {
1082                 iommu->iommu_dev = iommu_device_create(NULL, iommu,
1083                                                        intel_iommu_groups,
1084                                                        "%s", iommu->name);
1085
1086                 if (IS_ERR(iommu->iommu_dev)) {
1087                         err = PTR_ERR(iommu->iommu_dev);
1088                         goto err_unmap;
1089                 }
1090         }
1091
1092         drhd->iommu = iommu;
1093
1094         return 0;
1095
1096 err_unmap:
1097         unmap_iommu(iommu);
1098 error_free_seq_id:
1099         dmar_free_seq_id(iommu);
1100 error:
1101         kfree(iommu);
1102         return err;
1103 }
1104
1105 static void free_iommu(struct intel_iommu *iommu)
1106 {
1107         iommu_device_destroy(iommu->iommu_dev);
1108
1109         if (iommu->irq) {
1110                 if (iommu->pr_irq) {
1111                         free_irq(iommu->pr_irq, iommu);
1112                         dmar_free_hwirq(iommu->pr_irq);
1113                         iommu->pr_irq = 0;
1114                 }
1115                 free_irq(iommu->irq, iommu);
1116                 dmar_free_hwirq(iommu->irq);
1117                 iommu->irq = 0;
1118         }
1119
1120         if (iommu->qi) {
1121                 free_page((unsigned long)iommu->qi->desc);
1122                 kfree(iommu->qi->desc_status);
1123                 kfree(iommu->qi);
1124         }
1125
1126         if (iommu->reg)
1127                 unmap_iommu(iommu);
1128
1129         dmar_free_seq_id(iommu);
1130         kfree(iommu);
1131 }
1132
1133 /*
1134  * Reclaim all the submitted descriptors which have completed its work.
1135  */
1136 static inline void reclaim_free_desc(struct q_inval *qi)
1137 {
1138         while (qi->desc_status[qi->free_tail] == QI_DONE ||
1139                qi->desc_status[qi->free_tail] == QI_ABORT) {
1140                 qi->desc_status[qi->free_tail] = QI_FREE;
1141                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1142                 qi->free_cnt++;
1143         }
1144 }
1145
1146 static int qi_check_fault(struct intel_iommu *iommu, int index)
1147 {
1148         u32 fault;
1149         int head, tail;
1150         struct q_inval *qi = iommu->qi;
1151         int wait_index = (index + 1) % QI_LENGTH;
1152
1153         if (qi->desc_status[wait_index] == QI_ABORT)
1154                 return -EAGAIN;
1155
1156         fault = readl(iommu->reg + DMAR_FSTS_REG);
1157
1158         /*
1159          * If IQE happens, the head points to the descriptor associated
1160          * with the error. No new descriptors are fetched until the IQE
1161          * is cleared.
1162          */
1163         if (fault & DMA_FSTS_IQE) {
1164                 head = readl(iommu->reg + DMAR_IQH_REG);
1165                 if ((head >> DMAR_IQ_SHIFT) == index) {
1166                         pr_err("VT-d detected invalid descriptor: "
1167                                 "low=%llx, high=%llx\n",
1168                                 (unsigned long long)qi->desc[index].low,
1169                                 (unsigned long long)qi->desc[index].high);
1170                         memcpy(&qi->desc[index], &qi->desc[wait_index],
1171                                         sizeof(struct qi_desc));
1172                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1173                         return -EINVAL;
1174                 }
1175         }
1176
1177         /*
1178          * If ITE happens, all pending wait_desc commands are aborted.
1179          * No new descriptors are fetched until the ITE is cleared.
1180          */
1181         if (fault & DMA_FSTS_ITE) {
1182                 head = readl(iommu->reg + DMAR_IQH_REG);
1183                 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1184                 head |= 1;
1185                 tail = readl(iommu->reg + DMAR_IQT_REG);
1186                 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1187
1188                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1189
1190                 do {
1191                         if (qi->desc_status[head] == QI_IN_USE)
1192                                 qi->desc_status[head] = QI_ABORT;
1193                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1194                 } while (head != tail);
1195
1196                 if (qi->desc_status[wait_index] == QI_ABORT)
1197                         return -EAGAIN;
1198         }
1199
1200         if (fault & DMA_FSTS_ICE)
1201                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1202
1203         return 0;
1204 }
1205
1206 /*
1207  * Submit the queued invalidation descriptor to the remapping
1208  * hardware unit and wait for its completion.
1209  */
1210 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1211 {
1212         int rc;
1213         struct q_inval *qi = iommu->qi;
1214         struct qi_desc *hw, wait_desc;
1215         int wait_index, index;
1216         unsigned long flags;
1217
1218         if (!qi)
1219                 return 0;
1220
1221         hw = qi->desc;
1222
1223 restart:
1224         rc = 0;
1225
1226         raw_spin_lock_irqsave(&qi->q_lock, flags);
1227         while (qi->free_cnt < 3) {
1228                 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1229                 cpu_relax();
1230                 raw_spin_lock_irqsave(&qi->q_lock, flags);
1231         }
1232
1233         index = qi->free_head;
1234         wait_index = (index + 1) % QI_LENGTH;
1235
1236         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1237
1238         hw[index] = *desc;
1239
1240         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1241                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1242         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1243
1244         hw[wait_index] = wait_desc;
1245
1246         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1247         qi->free_cnt -= 2;
1248
1249         /*
1250          * update the HW tail register indicating the presence of
1251          * new descriptors.
1252          */
1253         writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
1254
1255         while (qi->desc_status[wait_index] != QI_DONE) {
1256                 /*
1257                  * We will leave the interrupts disabled, to prevent interrupt
1258                  * context to queue another cmd while a cmd is already submitted
1259                  * and waiting for completion on this cpu. This is to avoid
1260                  * a deadlock where the interrupt context can wait indefinitely
1261                  * for free slots in the queue.
1262                  */
1263                 rc = qi_check_fault(iommu, index);
1264                 if (rc)
1265                         break;
1266
1267                 raw_spin_unlock(&qi->q_lock);
1268                 cpu_relax();
1269                 raw_spin_lock(&qi->q_lock);
1270         }
1271
1272         qi->desc_status[index] = QI_DONE;
1273
1274         reclaim_free_desc(qi);
1275         raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1276
1277         if (rc == -EAGAIN)
1278                 goto restart;
1279
1280         return rc;
1281 }
1282
1283 /*
1284  * Flush the global interrupt entry cache.
1285  */
1286 void qi_global_iec(struct intel_iommu *iommu)
1287 {
1288         struct qi_desc desc;
1289
1290         desc.low = QI_IEC_TYPE;
1291         desc.high = 0;
1292
1293         /* should never fail */
1294         qi_submit_sync(&desc, iommu);
1295 }
1296
1297 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1298                       u64 type)
1299 {
1300         struct qi_desc desc;
1301
1302         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1303                         | QI_CC_GRAN(type) | QI_CC_TYPE;
1304         desc.high = 0;
1305
1306         qi_submit_sync(&desc, iommu);
1307 }
1308
1309 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1310                     unsigned int size_order, u64 type)
1311 {
1312         u8 dw = 0, dr = 0;
1313
1314         struct qi_desc desc;
1315         int ih = 0;
1316
1317         if (cap_write_drain(iommu->cap))
1318                 dw = 1;
1319
1320         if (cap_read_drain(iommu->cap))
1321                 dr = 1;
1322
1323         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1324                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1325         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1326                 | QI_IOTLB_AM(size_order);
1327
1328         qi_submit_sync(&desc, iommu);
1329 }
1330
1331 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1332                         u64 addr, unsigned mask)
1333 {
1334         struct qi_desc desc;
1335
1336         if (mask) {
1337                 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1338                 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1339                 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1340         } else
1341                 desc.high = QI_DEV_IOTLB_ADDR(addr);
1342
1343         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1344                 qdep = 0;
1345
1346         desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1347                    QI_DIOTLB_TYPE;
1348
1349         qi_submit_sync(&desc, iommu);
1350 }
1351
1352 /*
1353  * Disable Queued Invalidation interface.
1354  */
1355 void dmar_disable_qi(struct intel_iommu *iommu)
1356 {
1357         unsigned long flags;
1358         u32 sts;
1359         cycles_t start_time = get_cycles();
1360
1361         if (!ecap_qis(iommu->ecap))
1362                 return;
1363
1364         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1365
1366         sts =  readl(iommu->reg + DMAR_GSTS_REG);
1367         if (!(sts & DMA_GSTS_QIES))
1368                 goto end;
1369
1370         /*
1371          * Give a chance to HW to complete the pending invalidation requests.
1372          */
1373         while ((readl(iommu->reg + DMAR_IQT_REG) !=
1374                 readl(iommu->reg + DMAR_IQH_REG)) &&
1375                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1376                 cpu_relax();
1377
1378         iommu->gcmd &= ~DMA_GCMD_QIE;
1379         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1380
1381         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1382                       !(sts & DMA_GSTS_QIES), sts);
1383 end:
1384         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1385 }
1386
1387 /*
1388  * Enable queued invalidation.
1389  */
1390 static void __dmar_enable_qi(struct intel_iommu *iommu)
1391 {
1392         u32 sts;
1393         unsigned long flags;
1394         struct q_inval *qi = iommu->qi;
1395
1396         qi->free_head = qi->free_tail = 0;
1397         qi->free_cnt = QI_LENGTH;
1398
1399         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1400
1401         /* write zero to the tail reg */
1402         writel(0, iommu->reg + DMAR_IQT_REG);
1403
1404         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1405
1406         iommu->gcmd |= DMA_GCMD_QIE;
1407         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1408
1409         /* Make sure hardware complete it */
1410         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1411
1412         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1413 }
1414
1415 /*
1416  * Enable Queued Invalidation interface. This is a must to support
1417  * interrupt-remapping. Also used by DMA-remapping, which replaces
1418  * register based IOTLB invalidation.
1419  */
1420 int dmar_enable_qi(struct intel_iommu *iommu)
1421 {
1422         struct q_inval *qi;
1423         struct page *desc_page;
1424
1425         if (!ecap_qis(iommu->ecap))
1426                 return -ENOENT;
1427
1428         /*
1429          * queued invalidation is already setup and enabled.
1430          */
1431         if (iommu->qi)
1432                 return 0;
1433
1434         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1435         if (!iommu->qi)
1436                 return -ENOMEM;
1437
1438         qi = iommu->qi;
1439
1440
1441         desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1442         if (!desc_page) {
1443                 kfree(qi);
1444                 iommu->qi = NULL;
1445                 return -ENOMEM;
1446         }
1447
1448         qi->desc = page_address(desc_page);
1449
1450         qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1451         if (!qi->desc_status) {
1452                 free_page((unsigned long) qi->desc);
1453                 kfree(qi);
1454                 iommu->qi = NULL;
1455                 return -ENOMEM;
1456         }
1457
1458         raw_spin_lock_init(&qi->q_lock);
1459
1460         __dmar_enable_qi(iommu);
1461
1462         return 0;
1463 }
1464
1465 /* iommu interrupt handling. Most stuff are MSI-like. */
1466
1467 enum faulttype {
1468         DMA_REMAP,
1469         INTR_REMAP,
1470         UNKNOWN,
1471 };
1472
1473 static const char *dma_remap_fault_reasons[] =
1474 {
1475         "Software",
1476         "Present bit in root entry is clear",
1477         "Present bit in context entry is clear",
1478         "Invalid context entry",
1479         "Access beyond MGAW",
1480         "PTE Write access is not set",
1481         "PTE Read access is not set",
1482         "Next page table ptr is invalid",
1483         "Root table address invalid",
1484         "Context table ptr is invalid",
1485         "non-zero reserved fields in RTP",
1486         "non-zero reserved fields in CTP",
1487         "non-zero reserved fields in PTE",
1488         "PCE for translation request specifies blocking",
1489 };
1490
1491 static const char *irq_remap_fault_reasons[] =
1492 {
1493         "Detected reserved fields in the decoded interrupt-remapped request",
1494         "Interrupt index exceeded the interrupt-remapping table size",
1495         "Present field in the IRTE entry is clear",
1496         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1497         "Detected reserved fields in the IRTE entry",
1498         "Blocked a compatibility format interrupt request",
1499         "Blocked an interrupt request due to source-id verification failure",
1500 };
1501
1502 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1503 {
1504         if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1505                                         ARRAY_SIZE(irq_remap_fault_reasons))) {
1506                 *fault_type = INTR_REMAP;
1507                 return irq_remap_fault_reasons[fault_reason - 0x20];
1508         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1509                 *fault_type = DMA_REMAP;
1510                 return dma_remap_fault_reasons[fault_reason];
1511         } else {
1512                 *fault_type = UNKNOWN;
1513                 return "Unknown";
1514         }
1515 }
1516
1517
1518 static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1519 {
1520         if (iommu->irq == irq)
1521                 return DMAR_FECTL_REG;
1522         else if (iommu->pr_irq == irq)
1523                 return DMAR_PECTL_REG;
1524         else
1525                 BUG();
1526 }
1527
1528 void dmar_msi_unmask(struct irq_data *data)
1529 {
1530         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1531         int reg = dmar_msi_reg(iommu, data->irq);
1532         unsigned long flag;
1533
1534         /* unmask it */
1535         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1536         writel(0, iommu->reg + reg);
1537         /* Read a reg to force flush the post write */
1538         readl(iommu->reg + reg);
1539         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1540 }
1541
1542 void dmar_msi_mask(struct irq_data *data)
1543 {
1544         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1545         int reg = dmar_msi_reg(iommu, data->irq);
1546         unsigned long flag;
1547
1548         /* mask it */
1549         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1550         writel(DMA_FECTL_IM, iommu->reg + reg);
1551         /* Read a reg to force flush the post write */
1552         readl(iommu->reg + reg);
1553         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1554 }
1555
1556 void dmar_msi_write(int irq, struct msi_msg *msg)
1557 {
1558         struct intel_iommu *iommu = irq_get_handler_data(irq);
1559         int reg = dmar_msi_reg(iommu, irq);
1560         unsigned long flag;
1561
1562         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1563         writel(msg->data, iommu->reg + reg + 4);
1564         writel(msg->address_lo, iommu->reg + reg + 8);
1565         writel(msg->address_hi, iommu->reg + reg + 12);
1566         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1567 }
1568
1569 void dmar_msi_read(int irq, struct msi_msg *msg)
1570 {
1571         struct intel_iommu *iommu = irq_get_handler_data(irq);
1572         int reg = dmar_msi_reg(iommu, irq);
1573         unsigned long flag;
1574
1575         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1576         msg->data = readl(iommu->reg + reg + 4);
1577         msg->address_lo = readl(iommu->reg + reg + 8);
1578         msg->address_hi = readl(iommu->reg + reg + 12);
1579         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1580 }
1581
1582 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1583                 u8 fault_reason, u16 source_id, unsigned long long addr)
1584 {
1585         const char *reason;
1586         int fault_type;
1587
1588         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1589
1590         if (fault_type == INTR_REMAP)
1591                 pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index %llx [fault reason %02d] %s\n",
1592                         source_id >> 8, PCI_SLOT(source_id & 0xFF),
1593                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1594                         fault_reason, reason);
1595         else
1596                 pr_err("[%s] Request device [%02x:%02x.%d] fault addr %llx [fault reason %02d] %s\n",
1597                        type ? "DMA Read" : "DMA Write",
1598                        source_id >> 8, PCI_SLOT(source_id & 0xFF),
1599                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1600         return 0;
1601 }
1602
1603 #define PRIMARY_FAULT_REG_LEN (16)
1604 irqreturn_t dmar_fault(int irq, void *dev_id)
1605 {
1606         struct intel_iommu *iommu = dev_id;
1607         int reg, fault_index;
1608         u32 fault_status;
1609         unsigned long flag;
1610         bool ratelimited;
1611         static DEFINE_RATELIMIT_STATE(rs,
1612                                       DEFAULT_RATELIMIT_INTERVAL,
1613                                       DEFAULT_RATELIMIT_BURST);
1614
1615         /* Disable printing, simply clear the fault when ratelimited */
1616         ratelimited = !__ratelimit(&rs);
1617
1618         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1619         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1620         if (fault_status && !ratelimited)
1621                 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1622
1623         /* TBD: ignore advanced fault log currently */
1624         if (!(fault_status & DMA_FSTS_PPF))
1625                 goto unlock_exit;
1626
1627         fault_index = dma_fsts_fault_record_index(fault_status);
1628         reg = cap_fault_reg_offset(iommu->cap);
1629         while (1) {
1630                 u8 fault_reason;
1631                 u16 source_id;
1632                 u64 guest_addr;
1633                 int type;
1634                 u32 data;
1635
1636                 /* highest 32 bits */
1637                 data = readl(iommu->reg + reg +
1638                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1639                 if (!(data & DMA_FRCD_F))
1640                         break;
1641
1642                 if (!ratelimited) {
1643                         fault_reason = dma_frcd_fault_reason(data);
1644                         type = dma_frcd_type(data);
1645
1646                         data = readl(iommu->reg + reg +
1647                                      fault_index * PRIMARY_FAULT_REG_LEN + 8);
1648                         source_id = dma_frcd_source_id(data);
1649
1650                         guest_addr = dmar_readq(iommu->reg + reg +
1651                                         fault_index * PRIMARY_FAULT_REG_LEN);
1652                         guest_addr = dma_frcd_page_addr(guest_addr);
1653                 }
1654
1655                 /* clear the fault */
1656                 writel(DMA_FRCD_F, iommu->reg + reg +
1657                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1658
1659                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1660
1661                 if (!ratelimited)
1662                         dmar_fault_do_one(iommu, type, fault_reason,
1663                                           source_id, guest_addr);
1664
1665                 fault_index++;
1666                 if (fault_index >= cap_num_fault_regs(iommu->cap))
1667                         fault_index = 0;
1668                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1669         }
1670
1671         writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1672
1673 unlock_exit:
1674         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1675         return IRQ_HANDLED;
1676 }
1677
1678 int dmar_set_interrupt(struct intel_iommu *iommu)
1679 {
1680         int irq, ret;
1681
1682         /*
1683          * Check if the fault interrupt is already initialized.
1684          */
1685         if (iommu->irq)
1686                 return 0;
1687
1688         irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
1689         if (irq > 0) {
1690                 iommu->irq = irq;
1691         } else {
1692                 pr_err("No free IRQ vectors\n");
1693                 return -EINVAL;
1694         }
1695
1696         ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1697         if (ret)
1698                 pr_err("Can't request irq\n");
1699         return ret;
1700 }
1701
1702 int __init enable_drhd_fault_handling(void)
1703 {
1704         struct dmar_drhd_unit *drhd;
1705         struct intel_iommu *iommu;
1706
1707         /*
1708          * Enable fault control interrupt.
1709          */
1710         for_each_iommu(iommu, drhd) {
1711                 u32 fault_status;
1712                 int ret = dmar_set_interrupt(iommu);
1713
1714                 if (ret) {
1715                         pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1716                                (unsigned long long)drhd->reg_base_addr, ret);
1717                         return -1;
1718                 }
1719
1720                 /*
1721                  * Clear any previous faults.
1722                  */
1723                 dmar_fault(iommu->irq, iommu);
1724                 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1725                 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1726         }
1727
1728         return 0;
1729 }
1730
1731 /*
1732  * Re-enable Queued Invalidation interface.
1733  */
1734 int dmar_reenable_qi(struct intel_iommu *iommu)
1735 {
1736         if (!ecap_qis(iommu->ecap))
1737                 return -ENOENT;
1738
1739         if (!iommu->qi)
1740                 return -ENOENT;
1741
1742         /*
1743          * First disable queued invalidation.
1744          */
1745         dmar_disable_qi(iommu);
1746         /*
1747          * Then enable queued invalidation again. Since there is no pending
1748          * invalidation requests now, it's safe to re-enable queued
1749          * invalidation.
1750          */
1751         __dmar_enable_qi(iommu);
1752
1753         return 0;
1754 }
1755
1756 /*
1757  * Check interrupt remapping support in DMAR table description.
1758  */
1759 int __init dmar_ir_support(void)
1760 {
1761         struct acpi_table_dmar *dmar;
1762         dmar = (struct acpi_table_dmar *)dmar_tbl;
1763         if (!dmar)
1764                 return 0;
1765         return dmar->flags & 0x1;
1766 }
1767
1768 /* Check whether DMAR units are in use */
1769 static inline bool dmar_in_use(void)
1770 {
1771         return irq_remapping_enabled || intel_iommu_enabled;
1772 }
1773
1774 static int __init dmar_free_unused_resources(void)
1775 {
1776         struct dmar_drhd_unit *dmaru, *dmaru_n;
1777
1778         if (dmar_in_use())
1779                 return 0;
1780
1781         if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1782                 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1783
1784         down_write(&dmar_global_lock);
1785         list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1786                 list_del(&dmaru->list);
1787                 dmar_free_drhd(dmaru);
1788         }
1789         up_write(&dmar_global_lock);
1790
1791         return 0;
1792 }
1793
1794 late_initcall(dmar_free_unused_resources);
1795 IOMMU_INIT_POST(detect_intel_iommu);
1796
1797 /*
1798  * DMAR Hotplug Support
1799  * For more details, please refer to Intel(R) Virtualization Technology
1800  * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
1801  * "Remapping Hardware Unit Hot Plug".
1802  */
1803 static u8 dmar_hp_uuid[] = {
1804         /* 0000 */    0xA6, 0xA3, 0xC1, 0xD8, 0x9B, 0xBE, 0x9B, 0x4C,
1805         /* 0008 */    0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF
1806 };
1807
1808 /*
1809  * Currently there's only one revision and BIOS will not check the revision id,
1810  * so use 0 for safety.
1811  */
1812 #define DMAR_DSM_REV_ID                 0
1813 #define DMAR_DSM_FUNC_DRHD              1
1814 #define DMAR_DSM_FUNC_ATSR              2
1815 #define DMAR_DSM_FUNC_RHSA              3
1816
1817 static inline bool dmar_detect_dsm(acpi_handle handle, int func)
1818 {
1819         return acpi_check_dsm(handle, dmar_hp_uuid, DMAR_DSM_REV_ID, 1 << func);
1820 }
1821
1822 static int dmar_walk_dsm_resource(acpi_handle handle, int func,
1823                                   dmar_res_handler_t handler, void *arg)
1824 {
1825         int ret = -ENODEV;
1826         union acpi_object *obj;
1827         struct acpi_dmar_header *start;
1828         struct dmar_res_callback callback;
1829         static int res_type[] = {
1830                 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
1831                 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
1832                 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
1833         };
1834
1835         if (!dmar_detect_dsm(handle, func))
1836                 return 0;
1837
1838         obj = acpi_evaluate_dsm_typed(handle, dmar_hp_uuid, DMAR_DSM_REV_ID,
1839                                       func, NULL, ACPI_TYPE_BUFFER);
1840         if (!obj)
1841                 return -ENODEV;
1842
1843         memset(&callback, 0, sizeof(callback));
1844         callback.cb[res_type[func]] = handler;
1845         callback.arg[res_type[func]] = arg;
1846         start = (struct acpi_dmar_header *)obj->buffer.pointer;
1847         ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
1848
1849         ACPI_FREE(obj);
1850
1851         return ret;
1852 }
1853
1854 static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
1855 {
1856         int ret;
1857         struct dmar_drhd_unit *dmaru;
1858
1859         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1860         if (!dmaru)
1861                 return -ENODEV;
1862
1863         ret = dmar_ir_hotplug(dmaru, true);
1864         if (ret == 0)
1865                 ret = dmar_iommu_hotplug(dmaru, true);
1866
1867         return ret;
1868 }
1869
1870 static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
1871 {
1872         int i, ret;
1873         struct device *dev;
1874         struct dmar_drhd_unit *dmaru;
1875
1876         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1877         if (!dmaru)
1878                 return 0;
1879
1880         /*
1881          * All PCI devices managed by this unit should have been destroyed.
1882          */
1883         if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt) {
1884                 for_each_active_dev_scope(dmaru->devices,
1885                                           dmaru->devices_cnt, i, dev)
1886                         return -EBUSY;
1887         }
1888
1889         ret = dmar_ir_hotplug(dmaru, false);
1890         if (ret == 0)
1891                 ret = dmar_iommu_hotplug(dmaru, false);
1892
1893         return ret;
1894 }
1895
1896 static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
1897 {
1898         struct dmar_drhd_unit *dmaru;
1899
1900         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1901         if (dmaru) {
1902                 list_del_rcu(&dmaru->list);
1903                 synchronize_rcu();
1904                 dmar_free_drhd(dmaru);
1905         }
1906
1907         return 0;
1908 }
1909
1910 static int dmar_hotplug_insert(acpi_handle handle)
1911 {
1912         int ret;
1913         int drhd_count = 0;
1914
1915         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1916                                      &dmar_validate_one_drhd, (void *)1);
1917         if (ret)
1918                 goto out;
1919
1920         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1921                                      &dmar_parse_one_drhd, (void *)&drhd_count);
1922         if (ret == 0 && drhd_count == 0) {
1923                 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
1924                 goto out;
1925         } else if (ret) {
1926                 goto release_drhd;
1927         }
1928
1929         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
1930                                      &dmar_parse_one_rhsa, NULL);
1931         if (ret)
1932                 goto release_drhd;
1933
1934         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1935                                      &dmar_parse_one_atsr, NULL);
1936         if (ret)
1937                 goto release_atsr;
1938
1939         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1940                                      &dmar_hp_add_drhd, NULL);
1941         if (!ret)
1942                 return 0;
1943
1944         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1945                                &dmar_hp_remove_drhd, NULL);
1946 release_atsr:
1947         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1948                                &dmar_release_one_atsr, NULL);
1949 release_drhd:
1950         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1951                                &dmar_hp_release_drhd, NULL);
1952 out:
1953         return ret;
1954 }
1955
1956 static int dmar_hotplug_remove(acpi_handle handle)
1957 {
1958         int ret;
1959
1960         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1961                                      &dmar_check_one_atsr, NULL);
1962         if (ret)
1963                 return ret;
1964
1965         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1966                                      &dmar_hp_remove_drhd, NULL);
1967         if (ret == 0) {
1968                 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1969                                                &dmar_release_one_atsr, NULL));
1970                 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1971                                                &dmar_hp_release_drhd, NULL));
1972         } else {
1973                 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1974                                        &dmar_hp_add_drhd, NULL);
1975         }
1976
1977         return ret;
1978 }
1979
1980 static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
1981                                        void *context, void **retval)
1982 {
1983         acpi_handle *phdl = retval;
1984
1985         if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1986                 *phdl = handle;
1987                 return AE_CTRL_TERMINATE;
1988         }
1989
1990         return AE_OK;
1991 }
1992
1993 static int dmar_device_hotplug(acpi_handle handle, bool insert)
1994 {
1995         int ret;
1996         acpi_handle tmp = NULL;
1997         acpi_status status;
1998
1999         if (!dmar_in_use())
2000                 return 0;
2001
2002         if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2003                 tmp = handle;
2004         } else {
2005                 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
2006                                              ACPI_UINT32_MAX,
2007                                              dmar_get_dsm_handle,
2008                                              NULL, NULL, &tmp);
2009                 if (ACPI_FAILURE(status)) {
2010                         pr_warn("Failed to locate _DSM method.\n");
2011                         return -ENXIO;
2012                 }
2013         }
2014         if (tmp == NULL)
2015                 return 0;
2016
2017         down_write(&dmar_global_lock);
2018         if (insert)
2019                 ret = dmar_hotplug_insert(tmp);
2020         else
2021                 ret = dmar_hotplug_remove(tmp);
2022         up_write(&dmar_global_lock);
2023
2024         return ret;
2025 }
2026
2027 int dmar_device_add(acpi_handle handle)
2028 {
2029         return dmar_device_hotplug(handle, true);
2030 }
2031
2032 int dmar_device_remove(acpi_handle handle)
2033 {
2034         return dmar_device_hotplug(handle, false);
2035 }