virt/kvm/arm/vgic/vgic-init.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2015, 2016 ARM Ltd.
   4  */
   5
   6 #include <linux/uaccess.h>
   7 #include <linux/interrupt.h>
   8 #include <linux/cpu.h>
   9 #include <linux/kvm_host.h>
  10 #include <kvm/arm_vgic.h>
  11 #include <asm/kvm_emulate.h>
  12 #include <asm/kvm_mmu.h>
  13 #include "vgic.h"
  14
  15 /*
  16  * Initialization rules: there are multiple stages to the vgic
  17  * initialization, both for the distributor and the CPU interfaces.  The basic
  18  * idea is that even though the VGIC is not functional or not requested from
  19  * user space, the critical path of the run loop can still call VGIC functions
  20  * that just won't do anything, without them having to check additional
  21  * initialization flags to ensure they don't look at uninitialized data
  22  * structures.
  23  *
  24  * Distributor:
  25  *
  26  * - kvm_vgic_early_init(): initialization of static data that doesn't
  27  *   depend on any sizing information or emulation type. No allocation
  28  *   is allowed there.
  29  *
  30  * - vgic_init(): allocation and initialization of the generic data
  31  *   structures that depend on sizing information (number of CPUs,
  32  *   number of interrupts). Also initializes the vcpu specific data
  33  *   structures. Can be executed lazily for GICv2.
  34  *
  35  * CPU Interface:
  36  *
  37  * - kvm_vgic_vcpu_init(): initialization of static data that
  38  *   doesn't depend on any sizing information or emulation type. No
  39  *   allocation is allowed there.
  40  */
  41
  42 /* EARLY INIT */
  43
  44 /**
  45  * kvm_vgic_early_init() - Initialize static VGIC VCPU data structures
  46  * @kvm: The VM whose VGIC districutor should be initialized
  47  *
  48  * Only do initialization of static structures that don't require any
  49  * allocation or sizing information from userspace.  vgic_init() called
  50  * kvm_vgic_dist_init() which takes care of the rest.
  51  */
  52 void kvm_vgic_early_init(struct kvm *kvm)
  53 {
  54         struct vgic_dist *dist = &kvm->arch.vgic;
  55
  56         INIT_LIST_HEAD(&dist->lpi_list_head);
  57         raw_spin_lock_init(&dist->lpi_list_lock);
  58 }
  59
  60 /* CREATION */
  61
  62 /**
  63  * kvm_vgic_create: triggered by the instantiation of the VGIC device by
  64  * user space, either through the legacy KVM_CREATE_IRQCHIP ioctl (v2 only)
  65  * or through the generic KVM_CREATE_DEVICE API ioctl.
  66  * irqchip_in_kernel() tells you if this function succeeded or not.
  67  * @kvm: kvm struct pointer
  68  * @type: KVM_DEV_TYPE_ARM_VGIC_V[23]
  69  */
  70 int kvm_vgic_create(struct kvm *kvm, u32 type)
  71 {
  72         int i, vcpu_lock_idx = -1, ret;
  73         struct kvm_vcpu *vcpu;
  74
  75         if (irqchip_in_kernel(kvm))
  76                 return -EEXIST;
  77
  78         /*
  79          * This function is also called by the KVM_CREATE_IRQCHIP handler,
  80          * which had no chance yet to check the availability of the GICv2
  81          * emulation. So check this here again. KVM_CREATE_DEVICE does
  82          * the proper checks already.
  83          */
  84         if (type == KVM_DEV_TYPE_ARM_VGIC_V2 &&
  85                 !kvm_vgic_global_state.can_emulate_gicv2)
  86                 return -ENODEV;
  87
  88         /*
  89          * Any time a vcpu is run, vcpu_load is called which tries to grab the
  90          * vcpu->mutex.  By grabbing the vcpu->mutex of all VCPUs we ensure
  91          * that no other VCPUs are run while we create the vgic.
  92          */
  93         ret = -EBUSY;
  94         kvm_for_each_vcpu(i, vcpu, kvm) {
  95                 if (!mutex_trylock(&vcpu->mutex))
  96                         goto out_unlock;
  97                 vcpu_lock_idx = i;
  98         }
  99
 100         kvm_for_each_vcpu(i, vcpu, kvm) {
 101                 if (vcpu->arch.has_run_once)
 102                         goto out_unlock;
 103         }
 104         ret = 0;
 105
 106         if (type == KVM_DEV_TYPE_ARM_VGIC_V2)
 107                 kvm->arch.max_vcpus = VGIC_V2_MAX_CPUS;
 108         else
 109                 kvm->arch.max_vcpus = VGIC_V3_MAX_CPUS;
 110
 111         if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus) {
 112                 ret = -E2BIG;
 113                 goto out_unlock;
 114         }
 115
 116         kvm->arch.vgic.in_kernel = true;
 117         kvm->arch.vgic.vgic_model = type;
 118
 119         kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
 120
 121         if (type == KVM_DEV_TYPE_ARM_VGIC_V2)
 122                 kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
 123         else
 124                 INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions);
 125
 126 out_unlock:
 127         for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
 128                 vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
 129                 mutex_unlock(&vcpu->mutex);
 130         }
 131         return ret;
 132 }
 133
 134 /* INIT/DESTROY */
 135
 136 /**
 137  * kvm_vgic_dist_init: initialize the dist data structures
 138  * @kvm: kvm struct pointer
 139  * @nr_spis: number of spis, frozen by caller
 140  */
 141 static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis)
 142 {
 143         struct vgic_dist *dist = &kvm->arch.vgic;
 144         struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
 145         int i;
 146
 147         dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), GFP_KERNEL);
 148         if (!dist->spis)
 149                 return  -ENOMEM;
 150
 151         /*
 152          * In the following code we do not take the irq struct lock since
 153          * no other action on irq structs can happen while the VGIC is
 154          * not initialized yet:
 155          * If someone wants to inject an interrupt or does a MMIO access, we
 156          * require prior initialization in case of a virtual GICv3 or trigger
 157          * initialization when using a virtual GICv2.
 158          */
 159         for (i = 0; i < nr_spis; i++) {
 160                 struct vgic_irq *irq = &dist->spis[i];
 161
 162                 irq->intid = i + VGIC_NR_PRIVATE_IRQS;
 163                 INIT_LIST_HEAD(&irq->ap_list);
 164                 raw_spin_lock_init(&irq->irq_lock);
 165                 irq->vcpu = NULL;
 166                 irq->target_vcpu = vcpu0;
 167                 kref_init(&irq->refcount);
 168                 switch (dist->vgic_model) {
 169                 case KVM_DEV_TYPE_ARM_VGIC_V2:
 170                         irq->targets = 0;
 171                         irq->group = 0;
 172                         break;
 173                 case KVM_DEV_TYPE_ARM_VGIC_V3:
 174                         irq->mpidr = 0;
 175                         irq->group = 1;
 176                         break;
 177                 default:
 178                         kfree(dist->spis);
 179                         return -EINVAL;
 180                 }
 181         }
 182         return 0;
 183 }
 184
 185 /**
 186  * kvm_vgic_vcpu_init() - Initialize static VGIC VCPU data
 187  * structures and register VCPU-specific KVM iodevs
 188  *
 189  * @vcpu: pointer to the VCPU being created and initialized
 190  *
 191  * Only do initialization, but do not actually enable the
 192  * VGIC CPU interface
 193  */
 194 int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
 195 {
 196         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 197         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 198         int ret = 0;
 199         int i;
 200
 201         vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF;
 202         vgic_cpu->sgi_iodev.base_addr = VGIC_ADDR_UNDEF;
 203
 204         INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
 205         raw_spin_lock_init(&vgic_cpu->ap_list_lock);
 206
 207         /*
 208          * Enable and configure all SGIs to be edge-triggered and
 209          * configure all PPIs as level-triggered.
 210          */
 211         for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
 212                 struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
 213
 214                 INIT_LIST_HEAD(&irq->ap_list);
 215                 raw_spin_lock_init(&irq->irq_lock);
 216                 irq->intid = i;
 217                 irq->vcpu = NULL;
 218                 irq->target_vcpu = vcpu;
 219                 kref_init(&irq->refcount);
 220                 if (vgic_irq_is_sgi(i)) {
 221                         /* SGIs */
 222                         irq->enabled = 1;
 223                         irq->config = VGIC_CONFIG_EDGE;
 224                 } else {
 225                         /* PPIs */
 226                         irq->config = VGIC_CONFIG_LEVEL;
 227                 }
 228         }
 229
 230         if (!irqchip_in_kernel(vcpu->kvm))
 231                 return 0;
 232
 233         /*
 234          * If we are creating a VCPU with a GICv3 we must also register the
 235          * KVM io device for the redistributor that belongs to this VCPU.
 236          */
 237         if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
 238                 mutex_lock(&vcpu->kvm->lock);
 239                 ret = vgic_register_redist_iodev(vcpu);
 240                 mutex_unlock(&vcpu->kvm->lock);
 241         }
 242         return ret;
 243 }
 244
 245 static void kvm_vgic_vcpu_enable(struct kvm_vcpu *vcpu)
 246 {
 247         if (kvm_vgic_global_state.type == VGIC_V2)
 248                 vgic_v2_enable(vcpu);
 249         else
 250                 vgic_v3_enable(vcpu);
 251 }
 252
 253 /*
 254  * vgic_init: allocates and initializes dist and vcpu data structures
 255  * depending on two dimensioning parameters:
 256  * - the number of spis
 257  * - the number of vcpus
 258  * The function is generally called when nr_spis has been explicitly set
 259  * by the guest through the KVM DEVICE API. If not nr_spis is set to 256.
 260  * vgic_initialized() returns true when this function has succeeded.
 261  * Must be called with kvm->lock held!
 262  */
 263 int vgic_init(struct kvm *kvm)
 264 {
 265         struct vgic_dist *dist = &kvm->arch.vgic;
 266         struct kvm_vcpu *vcpu;
 267         int ret = 0, i, idx;
 268
 269         if (vgic_initialized(kvm))
 270                 return 0;
 271
 272         /* Are we also in the middle of creating a VCPU? */
 273         if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
 274                 return -EBUSY;
 275
 276         /* freeze the number of spis */
 277         if (!dist->nr_spis)
 278                 dist->nr_spis = VGIC_NR_IRQS_LEGACY - VGIC_NR_PRIVATE_IRQS;
 279
 280         ret = kvm_vgic_dist_init(kvm, dist->nr_spis);
 281         if (ret)
 282                 goto out;
 283
 284         /* Initialize groups on CPUs created before the VGIC type was known */
 285         kvm_for_each_vcpu(idx, vcpu, kvm) {
 286                 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 287
 288                 for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
 289                         struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
 290                         switch (dist->vgic_model) {
 291                         case KVM_DEV_TYPE_ARM_VGIC_V3:
 292                                 irq->group = 1;
 293                                 irq->mpidr = kvm_vcpu_get_mpidr_aff(vcpu);
 294                                 break;
 295                         case KVM_DEV_TYPE_ARM_VGIC_V2:
 296                                 irq->group = 0;
 297                                 irq->targets = 1U << idx;
 298                                 break;
 299                         default:
 300                                 ret = -EINVAL;
 301                                 goto out;
 302                         }
 303                 }
 304         }
 305
 306         if (vgic_has_its(kvm)) {
 307                 ret = vgic_v4_init(kvm);
 308                 if (ret)
 309                         goto out;
 310         }
 311
 312         kvm_for_each_vcpu(i, vcpu, kvm)
 313                 kvm_vgic_vcpu_enable(vcpu);
 314
 315         ret = kvm_vgic_setup_default_irq_routing(kvm);
 316         if (ret)
 317                 goto out;
 318
 319         vgic_debug_init(kvm);
 320
 321         dist->implementation_rev = 2;
 322         dist->initialized = true;
 323
 324 out:
 325         return ret;
 326 }
 327
 328 static void kvm_vgic_dist_destroy(struct kvm *kvm)
 329 {
 330         struct vgic_dist *dist = &kvm->arch.vgic;
 331         struct vgic_redist_region *rdreg, *next;
 332
 333         dist->ready = false;
 334         dist->initialized = false;
 335
 336         kfree(dist->spis);
 337         dist->spis = NULL;
 338         dist->nr_spis = 0;
 339
 340         if (kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
 341                 list_for_each_entry_safe(rdreg, next, &dist->rd_regions, list) {
 342                         list_del(&rdreg->list);
 343                         kfree(rdreg);
 344                 }
 345                 INIT_LIST_HEAD(&dist->rd_regions);
 346         }
 347
 348         if (vgic_supports_direct_msis(kvm))
 349                 vgic_v4_teardown(kvm);
 350 }
 351
 352 void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
 353 {
 354         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 355
 356         INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
 357 }
 358
 359 /* To be called with kvm->lock held */
 360 static void __kvm_vgic_destroy(struct kvm *kvm)
 361 {
 362         struct kvm_vcpu *vcpu;
 363         int i;
 364
 365         vgic_debug_destroy(kvm);
 366
 367         kvm_vgic_dist_destroy(kvm);
 368
 369         kvm_for_each_vcpu(i, vcpu, kvm)
 370                 kvm_vgic_vcpu_destroy(vcpu);
 371 }
 372
 373 void kvm_vgic_destroy(struct kvm *kvm)
 374 {
 375         mutex_lock(&kvm->lock);
 376         __kvm_vgic_destroy(kvm);
 377         mutex_unlock(&kvm->lock);
 378 }
 379
 380 /**
 381  * vgic_lazy_init: Lazy init is only allowed if the GIC exposed to the guest
 382  * is a GICv2. A GICv3 must be explicitly initialized by the guest using the
 383  * KVM_DEV_ARM_VGIC_GRP_CTRL KVM_DEVICE group.
 384  * @kvm: kvm struct pointer
 385  */
 386 int vgic_lazy_init(struct kvm *kvm)
 387 {
 388         int ret = 0;
 389
 390         if (unlikely(!vgic_initialized(kvm))) {
 391                 /*
 392                  * We only provide the automatic initialization of the VGIC
 393                  * for the legacy case of a GICv2. Any other type must
 394                  * be explicitly initialized once setup with the respective
 395                  * KVM device call.
 396                  */
 397                 if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2)
 398                         return -EBUSY;
 399
 400                 mutex_lock(&kvm->lock);
 401                 ret = vgic_init(kvm);
 402                 mutex_unlock(&kvm->lock);
 403         }
 404
 405         return ret;
 406 }
 407
 408 /* RESOURCE MAPPING */
 409
 410 /**
 411  * Map the MMIO regions depending on the VGIC model exposed to the guest
 412  * called on the first VCPU run.
 413  * Also map the virtual CPU interface into the VM.
 414  * v2/v3 derivatives call vgic_init if not already done.
 415  * vgic_ready() returns true if this function has succeeded.
 416  * @kvm: kvm struct pointer
 417  */
 418 int kvm_vgic_map_resources(struct kvm *kvm)
 419 {
 420         struct vgic_dist *dist = &kvm->arch.vgic;
 421         int ret = 0;
 422
 423         mutex_lock(&kvm->lock);
 424         if (!irqchip_in_kernel(kvm))
 425                 goto out;
 426
 427         if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2)
 428                 ret = vgic_v2_map_resources(kvm);
 429         else
 430                 ret = vgic_v3_map_resources(kvm);
 431
 432         if (ret)
 433                 __kvm_vgic_destroy(kvm);
 434
 435 out:
 436         mutex_unlock(&kvm->lock);
 437         return ret;
 438 }
 439
 440 /* GENERIC PROBE */
 441
 442 static int vgic_init_cpu_starting(unsigned int cpu)
 443 {
 444         enable_percpu_irq(kvm_vgic_global_state.maint_irq, 0);
 445         return 0;
 446 }
 447
 448
 449 static int vgic_init_cpu_dying(unsigned int cpu)
 450 {
 451         disable_percpu_irq(kvm_vgic_global_state.maint_irq);
 452         return 0;
 453 }
 454
 455 static irqreturn_t vgic_maintenance_handler(int irq, void *data)
 456 {
 457         /*
 458          * We cannot rely on the vgic maintenance interrupt to be
 459          * delivered synchronously. This means we can only use it to
 460          * exit the VM, and we perform the handling of EOIed
 461          * interrupts on the exit path (see vgic_fold_lr_state).
 462          */
 463         return IRQ_HANDLED;
 464 }
 465
 466 /**
 467  * kvm_vgic_init_cpu_hardware - initialize the GIC VE hardware
 468  *
 469  * For a specific CPU, initialize the GIC VE hardware.
 470  */
 471 void kvm_vgic_init_cpu_hardware(void)
 472 {
 473         BUG_ON(preemptible());
 474
 475         /*
 476          * We want to make sure the list registers start out clear so that we
 477          * only have the program the used registers.
 478          */
 479         if (kvm_vgic_global_state.type == VGIC_V2)
 480                 vgic_v2_init_lrs();
 481         else
 482                 kvm_call_hyp(__vgic_v3_init_lrs);
 483 }
 484
 485 /**
 486  * kvm_vgic_hyp_init: populates the kvm_vgic_global_state variable
 487  * according to the host GIC model. Accordingly calls either
 488  * vgic_v2/v3_probe which registers the KVM_DEVICE that can be
 489  * instantiated by a guest later on .
 490  */
 491 int kvm_vgic_hyp_init(void)
 492 {
 493         const struct gic_kvm_info *gic_kvm_info;
 494         int ret;
 495
 496         gic_kvm_info = gic_get_kvm_info();
 497         if (!gic_kvm_info)
 498                 return -ENODEV;
 499
 500         if (!gic_kvm_info->maint_irq) {
 501                 kvm_err("No vgic maintenance irq\n");
 502                 return -ENXIO;
 503         }
 504
 505         switch (gic_kvm_info->type) {
 506         case GIC_V2:
 507                 ret = vgic_v2_probe(gic_kvm_info);
 508                 break;
 509         case GIC_V3:
 510                 ret = vgic_v3_probe(gic_kvm_info);
 511                 if (!ret) {
 512                         static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif);
 513                         kvm_info("GIC system register CPU interface enabled\n");
 514                 }
 515                 break;
 516         default:
 517                 ret = -ENODEV;
 518         };
 519
 520         if (ret)
 521                 return ret;
 522
 523         kvm_vgic_global_state.maint_irq = gic_kvm_info->maint_irq;
 524         ret = request_percpu_irq(kvm_vgic_global_state.maint_irq,
 525                                  vgic_maintenance_handler,
 526                                  "vgic", kvm_get_running_vcpus());
 527         if (ret) {
 528                 kvm_err("Cannot register interrupt %d\n",
 529                         kvm_vgic_global_state.maint_irq);
 530                 return ret;
 531         }
 532
 533         ret = cpuhp_setup_state(CPUHP_AP_KVM_ARM_VGIC_INIT_STARTING,
 534                                 "kvm/arm/vgic:starting",
 535                                 vgic_init_cpu_starting, vgic_init_cpu_dying);
 536         if (ret) {
 537                 kvm_err("Cannot register vgic CPU notifier\n");
 538                 goto out_free_irq;
 539         }
 540
 541         kvm_info("vgic interrupt IRQ%d\n", kvm_vgic_global_state.maint_irq);
 542         return 0;
 543
 544 out_free_irq:
 545         free_percpu_irq(kvm_vgic_global_state.maint_irq,
 546                         kvm_get_running_vcpus());
 547         return ret;
 548 }