kernel/irq/irqdesc.c

   1 /*
   2  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
   3  * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
   4  *
   5  * This file contains the interrupt descriptor management code
   6  *
   7  * Detailed information is available in Documentation/DocBook/genericirq
   8  *
   9  */
  10 #include <linux/irq.h>
  11 #include <linux/slab.h>
  12 #include <linux/export.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/radix-tree.h>
  16 #include <linux/bitmap.h>
  17 #include <linux/irqdomain.h>
  18 #include <linux/sysfs.h>
  19
  20 #include "internals.h"
  21
  22 /*
  23  * lockdep: we want to handle all irq_desc locks as a single lock-class:
  24  */
  25 static struct lock_class_key irq_desc_lock_class;
  26
  27 #if defined(CONFIG_SMP)
  28 static int __init irq_affinity_setup(char *str)
  29 {
  30         zalloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT);
  31         cpulist_parse(str, irq_default_affinity);
  32         /*
  33          * Set at least the boot cpu. We don't want to end up with
  34          * bugreports caused by random comandline masks
  35          */
  36         cpumask_set_cpu(smp_processor_id(), irq_default_affinity);
  37         return 1;
  38 }
  39 __setup("irqaffinity=", irq_affinity_setup);
  40
  41 static void __init init_irq_default_affinity(void)
  42 {
  43 #ifdef CONFIG_CPUMASK_OFFSTACK
  44         if (!irq_default_affinity)
  45                 zalloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT);
  46 #endif
  47         if (cpumask_empty(irq_default_affinity))
  48                 cpumask_setall(irq_default_affinity);
  49 }
  50 #else
  51 static void __init init_irq_default_affinity(void)
  52 {
  53 }
  54 #endif
  55
  56 #ifdef CONFIG_SMP
  57 static int alloc_masks(struct irq_desc *desc, gfp_t gfp, int node)
  58 {
  59         if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity,
  60                                      gfp, node))
  61                 return -ENOMEM;
  62
  63 #ifdef CONFIG_GENERIC_PENDING_IRQ
  64         if (!zalloc_cpumask_var_node(&desc->pending_mask, gfp, node)) {
  65                 free_cpumask_var(desc->irq_common_data.affinity);
  66                 return -ENOMEM;
  67         }
  68 #endif
  69         return 0;
  70 }
  71
  72 static void desc_smp_init(struct irq_desc *desc, int node,
  73                           const struct cpumask *affinity)
  74 {
  75         if (!affinity)
  76                 affinity = irq_default_affinity;
  77         cpumask_copy(desc->irq_common_data.affinity, affinity);
  78
  79 #ifdef CONFIG_GENERIC_PENDING_IRQ
  80         cpumask_clear(desc->pending_mask);
  81 #endif
  82 #ifdef CONFIG_NUMA
  83         desc->irq_common_data.node = node;
  84 #endif
  85 }
  86
  87 #else
  88 static inline int
  89 alloc_masks(struct irq_desc *desc, gfp_t gfp, int node) { return 0; }
  90 static inline void
  91 desc_smp_init(struct irq_desc *desc, int node, const struct cpumask *affinity) { }
  92 #endif
  93
  94 static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,
  95                               const struct cpumask *affinity, struct module *owner)
  96 {
  97         int cpu;
  98
  99         desc->irq_common_data.handler_data = NULL;
 100         desc->irq_common_data.msi_desc = NULL;
 101
 102         desc->irq_data.common = &desc->irq_common_data;
 103         desc->irq_data.irq = irq;
 104         desc->irq_data.chip = &no_irq_chip;
 105         desc->irq_data.chip_data = NULL;
 106         irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS);
 107         irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
 108         desc->handle_irq = handle_bad_irq;
 109         desc->depth = 1;
 110         desc->irq_count = 0;
 111         desc->irqs_unhandled = 0;
 112         desc->name = NULL;
 113         desc->owner = owner;
 114         for_each_possible_cpu(cpu)
 115                 *per_cpu_ptr(desc->kstat_irqs, cpu) = 0;
 116         desc_smp_init(desc, node, affinity);
 117 }
 118
 119 int nr_irqs = NR_IRQS;
 120 EXPORT_SYMBOL_GPL(nr_irqs);
 121
 122 static DEFINE_MUTEX(sparse_irq_lock);
 123 static DECLARE_BITMAP(allocated_irqs, IRQ_BITMAP_BITS);
 124
 125 #ifdef CONFIG_SPARSE_IRQ
 126
 127 static void irq_kobj_release(struct kobject *kobj);
 128
 129 #ifdef CONFIG_SYSFS
 130 static struct kobject *irq_kobj_base;
 131
 132 #define IRQ_ATTR_RO(_name) \
 133 static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
 134
 135 static ssize_t per_cpu_count_show(struct kobject *kobj,
 136                                   struct kobj_attribute *attr, char *buf)
 137 {
 138         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
 139         int cpu, irq = desc->irq_data.irq;
 140         ssize_t ret = 0;
 141         char *p = "";
 142
 143         for_each_possible_cpu(cpu) {
 144                 unsigned int c = kstat_irqs_cpu(irq, cpu);
 145
 146                 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%u", p, c);
 147                 p = ",";
 148         }
 149
 150         ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n");
 151         return ret;
 152 }
 153 IRQ_ATTR_RO(per_cpu_count);
 154
 155 static ssize_t chip_name_show(struct kobject *kobj,
 156                               struct kobj_attribute *attr, char *buf)
 157 {
 158         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
 159         ssize_t ret = 0;
 160
 161         raw_spin_lock_irq(&desc->lock);
 162         if (desc->irq_data.chip && desc->irq_data.chip->name) {
 163                 ret = scnprintf(buf, PAGE_SIZE, "%s\n",
 164                                 desc->irq_data.chip->name);
 165         }
 166         raw_spin_unlock_irq(&desc->lock);
 167
 168         return ret;
 169 }
 170 IRQ_ATTR_RO(chip_name);
 171
 172 static ssize_t hwirq_show(struct kobject *kobj,
 173                           struct kobj_attribute *attr, char *buf)
 174 {
 175         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
 176         ssize_t ret = 0;
 177
 178         raw_spin_lock_irq(&desc->lock);
 179         if (desc->irq_data.domain)
 180                 ret = sprintf(buf, "%d\n", (int)desc->irq_data.hwirq);
 181         raw_spin_unlock_irq(&desc->lock);
 182
 183         return ret;
 184 }
 185 IRQ_ATTR_RO(hwirq);
 186
 187 static ssize_t type_show(struct kobject *kobj,
 188                          struct kobj_attribute *attr, char *buf)
 189 {
 190         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
 191         ssize_t ret = 0;
 192
 193         raw_spin_lock_irq(&desc->lock);
 194         ret = sprintf(buf, "%s\n",
 195                       irqd_is_level_type(&desc->irq_data) ? "level" : "edge");
 196         raw_spin_unlock_irq(&desc->lock);
 197
 198         return ret;
 199
 200 }
 201 IRQ_ATTR_RO(type);
 202
 203 static ssize_t name_show(struct kobject *kobj,
 204                          struct kobj_attribute *attr, char *buf)
 205 {
 206         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
 207         ssize_t ret = 0;
 208
 209         raw_spin_lock_irq(&desc->lock);
 210         if (desc->name)
 211                 ret = scnprintf(buf, PAGE_SIZE, "%s\n", desc->name);
 212         raw_spin_unlock_irq(&desc->lock);
 213
 214         return ret;
 215 }
 216 IRQ_ATTR_RO(name);
 217
 218 static ssize_t actions_show(struct kobject *kobj,
 219                             struct kobj_attribute *attr, char *buf)
 220 {
 221         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
 222         struct irqaction *action;
 223         ssize_t ret = 0;
 224         char *p = "";
 225
 226         raw_spin_lock_irq(&desc->lock);
 227         for (action = desc->action; action != NULL; action = action->next) {
 228                 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%s",
 229                                  p, action->name);
 230                 p = ",";
 231         }
 232         raw_spin_unlock_irq(&desc->lock);
 233
 234         if (ret)
 235                 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n");
 236
 237         return ret;
 238 }
 239 IRQ_ATTR_RO(actions);
 240
 241 static struct attribute *irq_attrs[] = {
 242         &per_cpu_count_attr.attr,
 243         &chip_name_attr.attr,
 244         &hwirq_attr.attr,
 245         &type_attr.attr,
 246         &name_attr.attr,
 247         &actions_attr.attr,
 248         NULL
 249 };
 250
 251 static struct kobj_type irq_kobj_type = {
 252         .release        = irq_kobj_release,
 253         .sysfs_ops      = &kobj_sysfs_ops,
 254         .default_attrs  = irq_attrs,
 255 };
 256
 257 static void irq_sysfs_add(int irq, struct irq_desc *desc)
 258 {
 259         if (irq_kobj_base) {
 260                 /*
 261                  * Continue even in case of failure as this is nothing
 262                  * crucial.
 263                  */
 264                 if (kobject_add(&desc->kobj, irq_kobj_base, "%d", irq))
 265                         pr_warn("Failed to add kobject for irq %d\n", irq);
 266         }
 267 }
 268
 269 static int __init irq_sysfs_init(void)
 270 {
 271         struct irq_desc *desc;
 272         int irq;
 273
 274         /* Prevent concurrent irq alloc/free */
 275         irq_lock_sparse();
 276
 277         irq_kobj_base = kobject_create_and_add("irq", kernel_kobj);
 278         if (!irq_kobj_base) {
 279                 irq_unlock_sparse();
 280                 return -ENOMEM;
 281         }
 282
 283         /* Add the already allocated interrupts */
 284         for_each_irq_desc(irq, desc)
 285                 irq_sysfs_add(irq, desc);
 286         irq_unlock_sparse();
 287
 288         return 0;
 289 }
 290 postcore_initcall(irq_sysfs_init);
 291
 292 #else /* !CONFIG_SYSFS */
 293
 294 static struct kobj_type irq_kobj_type = {
 295         .release        = irq_kobj_release,
 296 };
 297
 298 static void irq_sysfs_add(int irq, struct irq_desc *desc) {}
 299
 300 #endif /* CONFIG_SYSFS */
 301
 302 static RADIX_TREE(irq_desc_tree, GFP_KERNEL);
 303
 304 static void irq_insert_desc(unsigned int irq, struct irq_desc *desc)
 305 {
 306         radix_tree_insert(&irq_desc_tree, irq, desc);
 307 }
 308
 309 struct irq_desc *irq_to_desc(unsigned int irq)
 310 {
 311         return radix_tree_lookup(&irq_desc_tree, irq);
 312 }
 313 EXPORT_SYMBOL(irq_to_desc);
 314
 315 static void delete_irq_desc(unsigned int irq)
 316 {
 317         radix_tree_delete(&irq_desc_tree, irq);
 318 }
 319
 320 #ifdef CONFIG_SMP
 321 static void free_masks(struct irq_desc *desc)
 322 {
 323 #ifdef CONFIG_GENERIC_PENDING_IRQ
 324         free_cpumask_var(desc->pending_mask);
 325 #endif
 326         free_cpumask_var(desc->irq_common_data.affinity);
 327 }
 328 #else
 329 static inline void free_masks(struct irq_desc *desc) { }
 330 #endif
 331
 332 void irq_lock_sparse(void)
 333 {
 334         mutex_lock(&sparse_irq_lock);
 335 }
 336
 337 void irq_unlock_sparse(void)
 338 {
 339         mutex_unlock(&sparse_irq_lock);
 340 }
 341
 342 static struct irq_desc *alloc_desc(int irq, int node, unsigned int flags,
 343                                    const struct cpumask *affinity,
 344                                    struct module *owner)
 345 {
 346         struct irq_desc *desc;
 347         gfp_t gfp = GFP_KERNEL;
 348
 349         desc = kzalloc_node(sizeof(*desc), gfp, node);
 350         if (!desc)
 351                 return NULL;
 352         /* allocate based on nr_cpu_ids */
 353         desc->kstat_irqs = alloc_percpu(unsigned int);
 354         if (!desc->kstat_irqs)
 355                 goto err_desc;
 356
 357         if (alloc_masks(desc, gfp, node))
 358                 goto err_kstat;
 359
 360         raw_spin_lock_init(&desc->lock);
 361         lockdep_set_class(&desc->lock, &irq_desc_lock_class);
 362         init_rcu_head(&desc->rcu);
 363
 364         desc_set_defaults(irq, desc, node, affinity, owner);
 365         irqd_set(&desc->irq_data, flags);
 366         kobject_init(&desc->kobj, &irq_kobj_type);
 367
 368         return desc;
 369
 370 err_kstat:
 371         free_percpu(desc->kstat_irqs);
 372 err_desc:
 373         kfree(desc);
 374         return NULL;
 375 }
 376
 377 static void irq_kobj_release(struct kobject *kobj)
 378 {
 379         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
 380
 381         free_masks(desc);
 382         free_percpu(desc->kstat_irqs);
 383         kfree(desc);
 384 }
 385
 386 static void delayed_free_desc(struct rcu_head *rhp)
 387 {
 388         struct irq_desc *desc = container_of(rhp, struct irq_desc, rcu);
 389
 390         kobject_put(&desc->kobj);
 391 }
 392
 393 static void free_desc(unsigned int irq)
 394 {
 395         struct irq_desc *desc = irq_to_desc(irq);
 396
 397         unregister_irq_proc(irq, desc);
 398
 399         /*
 400          * sparse_irq_lock protects also show_interrupts() and
 401          * kstat_irq_usr(). Once we deleted the descriptor from the
 402          * sparse tree we can free it. Access in proc will fail to
 403          * lookup the descriptor.
 404          *
 405          * The sysfs entry must be serialized against a concurrent
 406          * irq_sysfs_init() as well.
 407          */
 408         kobject_del(&desc->kobj);
 409         delete_irq_desc(irq);
 410
 411         /*
 412          * We free the descriptor, masks and stat fields via RCU. That
 413          * allows demultiplex interrupts to do rcu based management of
 414          * the child interrupts.
 415          */
 416         call_rcu(&desc->rcu, delayed_free_desc);
 417 }
 418
 419 static int alloc_descs(unsigned int start, unsigned int cnt, int node,
 420                        const struct cpumask *affinity, struct module *owner)
 421 {
 422         const struct cpumask *mask = NULL;
 423         struct irq_desc *desc;
 424         unsigned int flags;
 425         int i;
 426
 427         /* Validate affinity mask(s) */
 428         if (affinity) {
 429                 for (i = 0, mask = affinity; i < cnt; i++, mask++) {
 430                         if (cpumask_empty(mask))
 431                                 return -EINVAL;
 432                 }
 433         }
 434
 435         flags = affinity ? IRQD_AFFINITY_MANAGED : 0;
 436         mask = NULL;
 437
 438         for (i = 0; i < cnt; i++) {
 439                 if (affinity) {
 440                         node = cpu_to_node(cpumask_first(affinity));
 441                         mask = affinity;
 442                         affinity++;
 443                 }
 444                 desc = alloc_desc(start + i, node, flags, mask, owner);
 445                 if (!desc)
 446                         goto err;
 447                 irq_insert_desc(start + i, desc);
 448                 irq_sysfs_add(start + i, desc);
 449         }
 450         bitmap_set(allocated_irqs, start, cnt);
 451         return start;
 452
 453 err:
 454         for (i--; i >= 0; i--)
 455                 free_desc(start + i);
 456         return -ENOMEM;
 457 }
 458
 459 static int irq_expand_nr_irqs(unsigned int nr)
 460 {
 461         if (nr > IRQ_BITMAP_BITS)
 462                 return -ENOMEM;
 463         nr_irqs = nr;
 464         return 0;
 465 }
 466
 467 int __init early_irq_init(void)
 468 {
 469         int i, initcnt, node = first_online_node;
 470         struct irq_desc *desc;
 471
 472         init_irq_default_affinity();
 473
 474         /* Let arch update nr_irqs and return the nr of preallocated irqs */
 475         initcnt = arch_probe_nr_irqs();
 476         printk(KERN_INFO "NR_IRQS:%d nr_irqs:%d %d\n", NR_IRQS, nr_irqs, initcnt);
 477
 478         if (WARN_ON(nr_irqs > IRQ_BITMAP_BITS))
 479                 nr_irqs = IRQ_BITMAP_BITS;
 480
 481         if (WARN_ON(initcnt > IRQ_BITMAP_BITS))
 482                 initcnt = IRQ_BITMAP_BITS;
 483
 484         if (initcnt > nr_irqs)
 485                 nr_irqs = initcnt;
 486
 487         for (i = 0; i < initcnt; i++) {
 488                 desc = alloc_desc(i, node, 0, NULL, NULL);
 489                 set_bit(i, allocated_irqs);
 490                 irq_insert_desc(i, desc);
 491         }
 492         return arch_early_irq_init();
 493 }
 494
 495 #else /* !CONFIG_SPARSE_IRQ */
 496
 497 struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
 498         [0 ... NR_IRQS-1] = {
 499                 .handle_irq     = handle_bad_irq,
 500                 .depth          = 1,
 501                 .lock           = __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock),
 502         }
 503 };
 504
 505 int __init early_irq_init(void)
 506 {
 507         int count, i, node = first_online_node;
 508         struct irq_desc *desc;
 509
 510         init_irq_default_affinity();
 511
 512         printk(KERN_INFO "NR_IRQS:%d\n", NR_IRQS);
 513
 514         desc = irq_desc;
 515         count = ARRAY_SIZE(irq_desc);
 516
 517         for (i = 0; i < count; i++) {
 518                 desc[i].kstat_irqs = alloc_percpu(unsigned int);
 519                 alloc_masks(&desc[i], GFP_KERNEL, node);
 520                 raw_spin_lock_init(&desc[i].lock);
 521                 lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
 522                 desc_set_defaults(i, &desc[i], node, NULL, NULL);
 523         }
 524         return arch_early_irq_init();
 525 }
 526
 527 struct irq_desc *irq_to_desc(unsigned int irq)
 528 {
 529         return (irq < NR_IRQS) ? irq_desc + irq : NULL;
 530 }
 531 EXPORT_SYMBOL(irq_to_desc);
 532
 533 static void free_desc(unsigned int irq)
 534 {
 535         struct irq_desc *desc = irq_to_desc(irq);
 536         unsigned long flags;
 537
 538         raw_spin_lock_irqsave(&desc->lock, flags);
 539         desc_set_defaults(irq, desc, irq_desc_get_node(desc), NULL, NULL);
 540         raw_spin_unlock_irqrestore(&desc->lock, flags);
 541 }
 542
 543 static inline int alloc_descs(unsigned int start, unsigned int cnt, int node,
 544                               const struct cpumask *affinity,
 545                               struct module *owner)
 546 {
 547         u32 i;
 548
 549         for (i = 0; i < cnt; i++) {
 550                 struct irq_desc *desc = irq_to_desc(start + i);
 551
 552                 desc->owner = owner;
 553         }
 554         bitmap_set(allocated_irqs, start, cnt);
 555         return start;
 556 }
 557
 558 static int irq_expand_nr_irqs(unsigned int nr)
 559 {
 560         return -ENOMEM;
 561 }
 562
 563 void irq_mark_irq(unsigned int irq)
 564 {
 565         mutex_lock(&sparse_irq_lock);
 566         bitmap_set(allocated_irqs, irq, 1);
 567         mutex_unlock(&sparse_irq_lock);
 568 }
 569
 570 #ifdef CONFIG_GENERIC_IRQ_LEGACY
 571 void irq_init_desc(unsigned int irq)
 572 {
 573         free_desc(irq);
 574 }
 575 #endif
 576
 577 #endif /* !CONFIG_SPARSE_IRQ */
 578
 579 /**
 580  * generic_handle_irq - Invoke the handler for a particular irq
 581  * @irq:        The irq number to handle
 582  *
 583  */
 584 int generic_handle_irq(unsigned int irq)
 585 {
 586         struct irq_desc *desc = irq_to_desc(irq);
 587
 588         if (!desc)
 589                 return -EINVAL;
 590         generic_handle_irq_desc(desc);
 591         return 0;
 592 }
 593 EXPORT_SYMBOL_GPL(generic_handle_irq);
 594
 595 #ifdef CONFIG_HANDLE_DOMAIN_IRQ
 596 /**
 597  * __handle_domain_irq - Invoke the handler for a HW irq belonging to a domain
 598  * @domain:     The domain where to perform the lookup
 599  * @hwirq:      The HW irq number to convert to a logical one
 600  * @lookup:     Whether to perform the domain lookup or not
 601  * @regs:       Register file coming from the low-level handling code
 602  *
 603  * Returns:     0 on success, or -EINVAL if conversion has failed
 604  */
 605 int __handle_domain_irq(struct irq_domain *domain, unsigned int hwirq,
 606                         bool lookup, struct pt_regs *regs)
 607 {
 608         struct pt_regs *old_regs = set_irq_regs(regs);
 609         unsigned int irq = hwirq;
 610         int ret = 0;
 611
 612         irq_enter();
 613
 614 #ifdef CONFIG_IRQ_DOMAIN
 615         if (lookup)
 616                 irq = irq_find_mapping(domain, hwirq);
 617 #endif
 618
 619         /*
 620          * Some hardware gives randomly wrong interrupts.  Rather
 621          * than crashing, do something sensible.
 622          */
 623         if (unlikely(!irq || irq >= nr_irqs)) {
 624                 ack_bad_irq(irq);
 625                 ret = -EINVAL;
 626         } else {
 627                 generic_handle_irq(irq);
 628         }
 629
 630         irq_exit();
 631         set_irq_regs(old_regs);
 632         return ret;
 633 }
 634 #endif
 635
 636 /* Dynamic interrupt handling */
 637
 638 /**
 639  * irq_free_descs - free irq descriptors
 640  * @from:       Start of descriptor range
 641  * @cnt:        Number of consecutive irqs to free
 642  */
 643 void irq_free_descs(unsigned int from, unsigned int cnt)
 644 {
 645         int i;
 646
 647         if (from >= nr_irqs || (from + cnt) > nr_irqs)
 648                 return;
 649
 650         mutex_lock(&sparse_irq_lock);
 651         for (i = 0; i < cnt; i++)
 652                 free_desc(from + i);
 653
 654         bitmap_clear(allocated_irqs, from, cnt);
 655         mutex_unlock(&sparse_irq_lock);
 656 }
 657 EXPORT_SYMBOL_GPL(irq_free_descs);
 658
 659 /**
 660  * irq_alloc_descs - allocate and initialize a range of irq descriptors
 661  * @irq:        Allocate for specific irq number if irq >= 0
 662  * @from:       Start the search from this irq number
 663  * @cnt:        Number of consecutive irqs to allocate.
 664  * @node:       Preferred node on which the irq descriptor should be allocated
 665  * @owner:      Owning module (can be NULL)
 666  * @affinity:   Optional pointer to an affinity mask array of size @cnt which
 667  *              hints where the irq descriptors should be allocated and which
 668  *              default affinities to use
 669  *
 670  * Returns the first irq number or error code
 671  */
 672 int __ref
 673 __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node,
 674                   struct module *owner, const struct cpumask *affinity)
 675 {
 676         int start, ret;
 677
 678         if (!cnt)
 679                 return -EINVAL;
 680
 681         if (irq >= 0) {
 682                 if (from > irq)
 683                         return -EINVAL;
 684                 from = irq;
 685         } else {
 686                 /*
 687                  * For interrupts which are freely allocated the
 688                  * architecture can force a lower bound to the @from
 689                  * argument. x86 uses this to exclude the GSI space.
 690                  */
 691                 from = arch_dynirq_lower_bound(from);
 692         }
 693
 694         mutex_lock(&sparse_irq_lock);
 695
 696         start = bitmap_find_next_zero_area(allocated_irqs, IRQ_BITMAP_BITS,
 697                                            from, cnt, 0);
 698         ret = -EEXIST;
 699         if (irq >=0 && start != irq)
 700                 goto unlock;
 701
 702         if (start + cnt > nr_irqs) {
 703                 ret = irq_expand_nr_irqs(start + cnt);
 704                 if (ret)
 705                         goto unlock;
 706         }
 707         ret = alloc_descs(start, cnt, node, affinity, owner);
 708 unlock:
 709         mutex_unlock(&sparse_irq_lock);
 710         return ret;
 711 }
 712 EXPORT_SYMBOL_GPL(__irq_alloc_descs);
 713
 714 #ifdef CONFIG_GENERIC_IRQ_LEGACY_ALLOC_HWIRQ
 715 /**
 716  * irq_alloc_hwirqs - Allocate an irq descriptor and initialize the hardware
 717  * @cnt:        number of interrupts to allocate
 718  * @node:       node on which to allocate
 719  *
 720  * Returns an interrupt number > 0 or 0, if the allocation fails.
 721  */
 722 unsigned int irq_alloc_hwirqs(int cnt, int node)
 723 {
 724         int i, irq = __irq_alloc_descs(-1, 0, cnt, node, NULL, NULL);
 725
 726         if (irq < 0)
 727                 return 0;
 728
 729         for (i = irq; cnt > 0; i++, cnt--) {
 730                 if (arch_setup_hwirq(i, node))
 731                         goto err;
 732                 irq_clear_status_flags(i, _IRQ_NOREQUEST);
 733         }
 734         return irq;
 735
 736 err:
 737         for (i--; i >= irq; i--) {
 738                 irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE);
 739                 arch_teardown_hwirq(i);
 740         }
 741         irq_free_descs(irq, cnt);
 742         return 0;
 743 }
 744 EXPORT_SYMBOL_GPL(irq_alloc_hwirqs);
 745
 746 /**
 747  * irq_free_hwirqs - Free irq descriptor and cleanup the hardware
 748  * @from:       Free from irq number
 749  * @cnt:        number of interrupts to free
 750  *
 751  */
 752 void irq_free_hwirqs(unsigned int from, int cnt)
 753 {
 754         int i, j;
 755
 756         for (i = from, j = cnt; j > 0; i++, j--) {
 757                 irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE);
 758                 arch_teardown_hwirq(i);
 759         }
 760         irq_free_descs(from, cnt);
 761 }
 762 EXPORT_SYMBOL_GPL(irq_free_hwirqs);
 763 #endif
 764
 765 /**
 766  * irq_get_next_irq - get next allocated irq number
 767  * @offset:     where to start the search
 768  *
 769  * Returns next irq number after offset or nr_irqs if none is found.
 770  */
 771 unsigned int irq_get_next_irq(unsigned int offset)
 772 {
 773         return find_next_bit(allocated_irqs, nr_irqs, offset);
 774 }
 775
 776 struct irq_desc *
 777 __irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
 778                     unsigned int check)
 779 {
 780         struct irq_desc *desc = irq_to_desc(irq);
 781
 782         if (desc) {
 783                 if (check & _IRQ_DESC_CHECK) {
 784                         if ((check & _IRQ_DESC_PERCPU) &&
 785                             !irq_settings_is_per_cpu_devid(desc))
 786                                 return NULL;
 787
 788                         if (!(check & _IRQ_DESC_PERCPU) &&
 789                             irq_settings_is_per_cpu_devid(desc))
 790                                 return NULL;
 791                 }
 792
 793                 if (bus)
 794                         chip_bus_lock(desc);
 795                 raw_spin_lock_irqsave(&desc->lock, *flags);
 796         }
 797         return desc;
 798 }
 799
 800 void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus)
 801 {
 802         raw_spin_unlock_irqrestore(&desc->lock, flags);
 803         if (bus)
 804                 chip_bus_sync_unlock(desc);
 805 }
 806
 807 int irq_set_percpu_devid_partition(unsigned int irq,
 808                                    const struct cpumask *affinity)
 809 {
 810         struct irq_desc *desc = irq_to_desc(irq);
 811
 812         if (!desc)
 813                 return -EINVAL;
 814
 815         if (desc->percpu_enabled)
 816                 return -EINVAL;
 817
 818         desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL);
 819
 820         if (!desc->percpu_enabled)
 821                 return -ENOMEM;
 822
 823         if (affinity)
 824                 desc->percpu_affinity = affinity;
 825         else
 826                 desc->percpu_affinity = cpu_possible_mask;
 827
 828         irq_set_percpu_devid_flags(irq);
 829         return 0;
 830 }
 831
 832 int irq_set_percpu_devid(unsigned int irq)
 833 {
 834         return irq_set_percpu_devid_partition(irq, NULL);
 835 }
 836
 837 int irq_get_percpu_devid_partition(unsigned int irq, struct cpumask *affinity)
 838 {
 839         struct irq_desc *desc = irq_to_desc(irq);
 840
 841         if (!desc || !desc->percpu_enabled)
 842                 return -EINVAL;
 843
 844         if (affinity)
 845                 cpumask_copy(affinity, desc->percpu_affinity);
 846
 847         return 0;
 848 }
 849
 850 void kstat_incr_irq_this_cpu(unsigned int irq)
 851 {
 852         kstat_incr_irqs_this_cpu(irq_to_desc(irq));
 853 }
 854
 855 /**
 856  * kstat_irqs_cpu - Get the statistics for an interrupt on a cpu
 857  * @irq:        The interrupt number
 858  * @cpu:        The cpu number
 859  *
 860  * Returns the sum of interrupt counts on @cpu since boot for
 861  * @irq. The caller must ensure that the interrupt is not removed
 862  * concurrently.
 863  */
 864 unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
 865 {
 866         struct irq_desc *desc = irq_to_desc(irq);
 867
 868         return desc && desc->kstat_irqs ?
 869                         *per_cpu_ptr(desc->kstat_irqs, cpu) : 0;
 870 }
 871
 872 /**
 873  * kstat_irqs - Get the statistics for an interrupt
 874  * @irq:        The interrupt number
 875  *
 876  * Returns the sum of interrupt counts on all cpus since boot for
 877  * @irq. The caller must ensure that the interrupt is not removed
 878  * concurrently.
 879  */
 880 unsigned int kstat_irqs(unsigned int irq)
 881 {
 882         struct irq_desc *desc = irq_to_desc(irq);
 883         int cpu;
 884         unsigned int sum = 0;
 885
 886         if (!desc || !desc->kstat_irqs)
 887                 return 0;
 888         for_each_possible_cpu(cpu)
 889                 sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
 890         return sum;
 891 }
 892
 893 /**
 894  * kstat_irqs_usr - Get the statistics for an interrupt
 895  * @irq:        The interrupt number
 896  *
 897  * Returns the sum of interrupt counts on all cpus since boot for
 898  * @irq. Contrary to kstat_irqs() this can be called from any
 899  * preemptible context. It's protected against concurrent removal of
 900  * an interrupt descriptor when sparse irqs are enabled.
 901  */
 902 unsigned int kstat_irqs_usr(unsigned int irq)
 903 {
 904         unsigned int sum;
 905
 906         irq_lock_sparse();
 907         sum = kstat_irqs(irq);
 908         irq_unlock_sparse();
 909         return sum;
 910 }