KVM: arm/arm64: The GIC is dead, long live the GIC

I don't think any single piece of the KVM/ARM code ever generated
as much hatred as the GIC emulation.

It was written by someone who had zero experience in modeling
hardware (me), was riddled with design flaws, should have been
scrapped and rewritten from scratch long before having a remote
chance of reaching mainline, and yet we supported it for a good
three years. No need to mention the names of those who suffered,
the git log is singing their praises.

Thankfully, we now have a much more maintainable implementation,
and we can safely put the grumpy old GIC to rest.

Fellow hackers, please raise your glass in memory of the GIC:

	The GIC is dead, long live the GIC!

Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>

1 files changed, 126 insertions, 255 deletions

diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index da0a524802cb..12640378db98 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -1,6 +1,5 @@
 /*
- * Copyright (C) 2012 ARM Ltd.
- * Author: Marc Zyngier <marc.zyngier@arm.com>
+ * Copyright (C) 2015, 2016 ARM Ltd.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -12,16 +11,10 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
-
-#ifndef __ASM_ARM_KVM_VGIC_H
-#define __ASM_ARM_KVM_VGIC_H
-
-#ifdef CONFIG_KVM_NEW_VGIC
-#include <kvm/vgic/vgic.h>
-#else
+#ifndef __KVM_ARM_VGIC_H
+#define __KVM_ARM_VGIC_H
 
 #include <linux/kernel.h>
 #include <linux/kvm.h>
@@ -29,248 +22,130 @@
 #include <linux/spinlock.h>
 #include <linux/types.h>
 #include <kvm/iodev.h>
-#include <linux/irqchip/arm-gic-common.h>
 
-#define VGIC_NR_IRQS_LEGACY	256
+#define VGIC_V3_MAX_CPUS	255
+#define VGIC_V2_MAX_CPUS	8
+#define VGIC_NR_IRQS_LEGACY     256
 #define VGIC_NR_SGIS		16
 #define VGIC_NR_PPIS		16
 #define VGIC_NR_PRIVATE_IRQS	(VGIC_NR_SGIS + VGIC_NR_PPIS)
+#define VGIC_MAX_PRIVATE	(VGIC_NR_PRIVATE_IRQS - 1)
+#define VGIC_MAX_SPI		1019
+#define VGIC_MAX_RESERVED	1023
+#define VGIC_MIN_LPI		8192
 
-#define VGIC_V2_MAX_LRS		(1 << 6)
-#define VGIC_V3_MAX_LRS		16
-#define VGIC_MAX_IRQS		1024
-#define VGIC_V2_MAX_CPUS	8
-#define VGIC_V3_MAX_CPUS	255
-
-#if (VGIC_NR_IRQS_LEGACY & 31)
-#error "VGIC_NR_IRQS must be a multiple of 32"
-#endif
+enum vgic_type {
+	VGIC_V2,		/* Good ol' GICv2 */
+	VGIC_V3,		/* New fancy GICv3 */
+};
 
-#if (VGIC_NR_IRQS_LEGACY > VGIC_MAX_IRQS)
-#error "VGIC_NR_IRQS must be <= 1024"
-#endif
+/* same for all guests, as depending only on the _host's_ GIC model */
+struct vgic_global {
+	/* type of the host GIC */
+	enum vgic_type		type;
 
-/*
- * The GIC distributor registers describing interrupts have two parts:
- * - 32 per-CPU interrupts (SGI + PPI)
- * - a bunch of shared interrupts (SPI)
- */
-struct vgic_bitmap {
-	/*
-	 * - One UL per VCPU for private interrupts (assumes UL is at
-	 *   least 32 bits)
-	 * - As many UL as necessary for shared interrupts.
-	 *
-	 * The private interrupts are accessed via the "private"
-	 * field, one UL per vcpu (the state for vcpu n is in
-	 * private[n]). The shared interrupts are accessed via the
-	 * "shared" pointer (IRQn state is at bit n-32 in the bitmap).
-	 */
-	unsigned long *private;
-	unsigned long *shared;
-};
+	/* Physical address of vgic virtual cpu interface */
+	phys_addr_t		vcpu_base;
 
-struct vgic_bytemap {
-	/*
-	 * - 8 u32 per VCPU for private interrupts
-	 * - As many u32 as necessary for shared interrupts.
-	 *
-	 * The private interrupts are accessed via the "private"
-	 * field, (the state for vcpu n is in private[n*8] to
-	 * private[n*8 + 7]). The shared interrupts are accessed via
-	 * the "shared" pointer (IRQn state is at byte (n-32)%4 of the
-	 * shared[(n-32)/4] word).
-	 */
-	u32 *private;
-	u32 *shared;
-};
+	/* virtual control interface mapping */
+	void __iomem		*vctrl_base;
 
-struct kvm_vcpu;
+	/* Number of implemented list registers */
+	int			nr_lr;
 
-enum vgic_type {
-	VGIC_V2,		/* Good ol' GICv2 */
-	VGIC_V3,		/* New fancy GICv3 */
-};
+	/* Maintenance IRQ number */
+	unsigned int		maint_irq;
 
-#define LR_STATE_PENDING	(1 << 0)
-#define LR_STATE_ACTIVE		(1 << 1)
-#define LR_STATE_MASK		(3 << 0)
-#define LR_EOI_INT		(1 << 2)
-#define LR_HW			(1 << 3)
+	/* maximum number of VCPUs allowed (GICv2 limits us to 8) */
+	int			max_gic_vcpus;
 
-struct vgic_lr {
-	unsigned irq:10;
-	union {
-		unsigned hwirq:10;
-		unsigned source:3;
-	};
-	unsigned state:4;
+	/* Only needed for the legacy KVM_CREATE_IRQCHIP */
+	bool			can_emulate_gicv2;
 };
 
-struct vgic_vmcr {
-	u32	ctlr;
-	u32	abpr;
-	u32	bpr;
-	u32	pmr;
-};
+extern struct vgic_global kvm_vgic_global_state;
 
-struct vgic_ops {
-	struct vgic_lr	(*get_lr)(const struct kvm_vcpu *, int);
-	void	(*set_lr)(struct kvm_vcpu *, int, struct vgic_lr);
-	u64	(*get_elrsr)(const struct kvm_vcpu *vcpu);
-	u64	(*get_eisr)(const struct kvm_vcpu *vcpu);
-	void	(*clear_eisr)(struct kvm_vcpu *vcpu);
-	u32	(*get_interrupt_status)(const struct kvm_vcpu *vcpu);
-	void	(*enable_underflow)(struct kvm_vcpu *vcpu);
-	void	(*disable_underflow)(struct kvm_vcpu *vcpu);
-	void	(*get_vmcr)(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
-	void	(*set_vmcr)(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
-	void	(*enable)(struct kvm_vcpu *vcpu);
-};
+#define VGIC_V2_MAX_LRS		(1 << 6)
+#define VGIC_V3_MAX_LRS		16
+#define VGIC_V3_LR_INDEX(lr)	(VGIC_V3_MAX_LRS - 1 - lr)
 
-struct vgic_params {
-	/* vgic type */
-	enum vgic_type	type;
-	/* Physical address of vgic virtual cpu interface */
-	phys_addr_t	vcpu_base;
-	/* Number of list registers */
-	u32		nr_lr;
-	/* Interrupt number */
-	unsigned int	maint_irq;
-	/* Virtual control interface base address */
-	void __iomem	*vctrl_base;
-	int		max_gic_vcpus;
-	/* Only needed for the legacy KVM_CREATE_IRQCHIP */
-	bool		can_emulate_gicv2;
+enum vgic_irq_config {
+	VGIC_CONFIG_EDGE = 0,
+	VGIC_CONFIG_LEVEL
 };
 
-struct vgic_vm_ops {
-	bool	(*queue_sgi)(struct kvm_vcpu *, int irq);
-	void	(*add_sgi_source)(struct kvm_vcpu *, int irq, int source);
-	int	(*init_model)(struct kvm *);
-	int	(*map_resources)(struct kvm *, const struct vgic_params *);
+struct vgic_irq {
+	spinlock_t irq_lock;		/* Protects the content of the struct */
+	struct list_head ap_list;
+
+	struct kvm_vcpu *vcpu;		/* SGIs and PPIs: The VCPU
+					 * SPIs and LPIs: The VCPU whose ap_list
+					 * this is queued on.
+					 */
+
+	struct kvm_vcpu *target_vcpu;	/* The VCPU that this interrupt should
+					 * be sent to, as a result of the
+					 * targets reg (v2) or the
+					 * affinity reg (v3).
+					 */
+
+	u32 intid;			/* Guest visible INTID */
+	bool pending;
+	bool line_level;		/* Level only */
+	bool soft_pending;		/* Level only */
+	bool active;			/* not used for LPIs */
+	bool enabled;
+	bool hw;			/* Tied to HW IRQ */
+	u32 hwintid;			/* HW INTID number */
+	union {
+		u8 targets;			/* GICv2 target VCPUs mask */
+		u32 mpidr;			/* GICv3 target VCPU */
+	};
+	u8 source;			/* GICv2 SGIs only */
+	u8 priority;
+	enum vgic_irq_config config;	/* Level or edge */
 };
 
+struct vgic_register_region;
+
 struct vgic_io_device {
-	gpa_t addr;
-	int len;
-	const struct vgic_io_range *reg_ranges;
+	gpa_t base_addr;
 	struct kvm_vcpu *redist_vcpu;
+	const struct vgic_register_region *regions;
+	int nr_regions;
 	struct kvm_io_device dev;
 };
 
-struct irq_phys_map {
-	u32			virt_irq;
-	u32			phys_irq;
-};
-
-struct irq_phys_map_entry {
-	struct list_head	entry;
-	struct rcu_head		rcu;
-	struct irq_phys_map	map;
-};
-
 struct vgic_dist {
-	spinlock_t		lock;
 	bool			in_kernel;
 	bool			ready;
+	bool			initialized;
 
 	/* vGIC model the kernel emulates for the guest (GICv2 or GICv3) */
 	u32			vgic_model;
 
-	int			nr_cpus;
-	int			nr_irqs;
+	int			nr_spis;
 
+	/* TODO: Consider moving to global state */
 	/* Virtual control interface mapping */
 	void __iomem		*vctrl_base;
 
-	/* Distributor and vcpu interface mapping in the guest */
-	phys_addr_t		vgic_dist_base;
-	/* GICv2 and GICv3 use different mapped register blocks */
+	/* base addresses in guest physical address space: */
+	gpa_t			vgic_dist_base;		/* distributor */
 	union {
-		phys_addr_t		vgic_cpu_base;
-		phys_addr_t		vgic_redist_base;
+		/* either a GICv2 CPU interface */
+		gpa_t			vgic_cpu_base;
+		/* or a number of GICv3 redistributor regions */
+		gpa_t			vgic_redist_base;
 	};
 
-	/* Distributor enabled */
-	u32			enabled;
-
-	/* Interrupt enabled (one bit per IRQ) */
-	struct vgic_bitmap	irq_enabled;
-
-	/* Level-triggered interrupt external input is asserted */
-	struct vgic_bitmap	irq_level;
-
-	/*
-	 * Interrupt state is pending on the distributor
-	 */
-	struct vgic_bitmap	irq_pending;
-
-	/*
-	 * Tracks writes to GICD_ISPENDRn and GICD_ICPENDRn for level-triggered
-	 * interrupts.  Essentially holds the state of the flip-flop in
-	 * Figure 4-10 on page 4-101 in ARM IHI 0048B.b.
-	 * Once set, it is only cleared for level-triggered interrupts on
-	 * guest ACKs (when we queue it) or writes to GICD_ICPENDRn.
-	 */
-	struct vgic_bitmap	irq_soft_pend;
-
-	/* Level-triggered interrupt queued on VCPU interface */
-	struct vgic_bitmap	irq_queued;
-
-	/* Interrupt was active when unqueue from VCPU interface */
-	struct vgic_bitmap	irq_active;
+	/* distributor enabled */
+	bool			enabled;
 
-	/* Interrupt priority. Not used yet. */
-	struct vgic_bytemap	irq_priority;
+	struct vgic_irq		*spis;
 
-	/* Level/edge triggered */
-	struct vgic_bitmap	irq_cfg;
-
-	/*
-	 * Source CPU per SGI and target CPU:
-	 *
-	 * Each byte represent a SGI observable on a VCPU, each bit of
-	 * this byte indicating if the corresponding VCPU has
-	 * generated this interrupt. This is a GICv2 feature only.
-	 *
-	 * For VCPUn (n < 8), irq_sgi_sources[n*16] to [n*16 + 15] are
-	 * the SGIs observable on VCPUn.
-	 */
-	u8			*irq_sgi_sources;
-
-	/*
-	 * Target CPU for each SPI:
-	 *
-	 * Array of available SPI, each byte indicating the target
-	 * VCPU for SPI. IRQn (n >=32) is at irq_spi_cpu[n-32].
-	 */
-	u8			*irq_spi_cpu;
-
-	/*
-	 * Reverse lookup of irq_spi_cpu for faster compute pending:
-	 *
-	 * Array of bitmaps, one per VCPU, describing if IRQn is
-	 * routed to a particular VCPU.
-	 */
-	struct vgic_bitmap	*irq_spi_target;
-
-	/* Target MPIDR for each IRQ (needed for GICv3 IROUTERn) only */
-	u32			*irq_spi_mpidr;
-
-	/* Bitmap indicating which CPU has something pending */
-	unsigned long		*irq_pending_on_cpu;
-
-	/* Bitmap indicating which CPU has active IRQs */
-	unsigned long		*irq_active_on_cpu;
-
-	struct vgic_vm_ops	vm_ops;
 	struct vgic_io_device	dist_iodev;
 	struct vgic_io_device	*redist_iodevs;
-
-	/* Virtual irq to hwirq mapping */
-	spinlock_t		irq_phys_map_lock;
-	struct list_head	irq_phys_map_list;
 };
 
 struct vgic_v2_cpu_if {
@@ -298,78 +173,74 @@ struct vgic_v3_cpu_if {
 };
 
 struct vgic_cpu {
-	/* Pending/active/both interrupts on this VCPU */
-	DECLARE_BITMAP(pending_percpu, VGIC_NR_PRIVATE_IRQS);
-	DECLARE_BITMAP(active_percpu, VGIC_NR_PRIVATE_IRQS);
-	DECLARE_BITMAP(pend_act_percpu, VGIC_NR_PRIVATE_IRQS);
-
-	/* Pending/active/both shared interrupts, dynamically sized */
-	unsigned long	*pending_shared;
-	unsigned long   *active_shared;
-	unsigned long   *pend_act_shared;
-
 	/* CPU vif control registers for world switch */
 	union {
 		struct vgic_v2_cpu_if	vgic_v2;
 		struct vgic_v3_cpu_if	vgic_v3;
 	};
 
-	/* Protected by the distributor's irq_phys_map_lock */
-	struct list_head	irq_phys_map_list;
-
-	u64		live_lrs;
-};
+	unsigned int used_lrs;
+	struct vgic_irq private_irqs[VGIC_NR_PRIVATE_IRQS];
 
-#define LR_EMPTY	0xff
+	spinlock_t ap_list_lock;	/* Protects the ap_list */
 
-#define INT_STATUS_EOI		(1 << 0)
-#define INT_STATUS_UNDERFLOW	(1 << 1)
+	/*
+	 * List of IRQs that this VCPU should consider because they are either
+	 * Active or Pending (hence the name; AP list), or because they recently
+	 * were one of the two and need to be migrated off this list to another
+	 * VCPU.
+	 */
+	struct list_head ap_list_head;
 
-struct kvm;
-struct kvm_vcpu;
+	u64 live_lrs;
+};
 
 int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write);
-int kvm_vgic_hyp_init(void);
-int kvm_vgic_map_resources(struct kvm *kvm);
-int kvm_vgic_get_max_vcpus(void);
 void kvm_vgic_early_init(struct kvm *kvm);
 int kvm_vgic_create(struct kvm *kvm, u32 type);
 void kvm_vgic_destroy(struct kvm *kvm);
 void kvm_vgic_vcpu_early_init(struct kvm_vcpu *vcpu);
 void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu);
-void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu);
-void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu);
-int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num,
+int kvm_vgic_map_resources(struct kvm *kvm);
+int kvm_vgic_hyp_init(void);
+
+int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int intid,
 			bool level);
-int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid,
-			       unsigned int virt_irq, bool level);
-void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg);
-int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu);
-int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, int virt_irq, int phys_irq);
+int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid, unsigned int intid,
+			       bool level);
+int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, u32 virt_irq, u32 phys_irq);
 int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int virt_irq);
 bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int virt_irq);
 
+int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu);
+
 #define irqchip_in_kernel(k)	(!!((k)->arch.vgic.in_kernel))
-#define vgic_initialized(k)	(!!((k)->arch.vgic.nr_cpus))
+#define vgic_initialized(k)	((k)->arch.vgic.initialized)
 #define vgic_ready(k)		((k)->arch.vgic.ready)
 #define vgic_valid_spi(k, i)	(((i) >= VGIC_NR_PRIVATE_IRQS) && \
-				 ((i) < (k)->arch.vgic.nr_irqs))
+			((i) < (k)->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS))
+
+bool kvm_vcpu_has_pending_irqs(struct kvm_vcpu *vcpu);
+void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu);
+void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu);
 
-int vgic_v2_probe(const struct gic_kvm_info *gic_kvm_info,
-		  const struct vgic_ops **ops,
-		  const struct vgic_params **params);
 #ifdef CONFIG_KVM_ARM_VGIC_V3
-int vgic_v3_probe(const struct gic_kvm_info *gic_kvm_info,
-		  const struct vgic_ops **ops,
-		  const struct vgic_params **params);
+void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg);
 #else
-static inline int vgic_v3_probe(const struct gic_kvm_info *gic_kvm_info,
-				const struct vgic_ops **ops,
-				const struct vgic_params **params)
+static inline void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg)
 {
-	return -ENODEV;
 }
 #endif
 
-#endif	/* old VGIC include */
-#endif
+/**
+ * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW
+ *
+ * The host's GIC naturally limits the maximum amount of VCPUs a guest
+ * can use.
+ */
+static inline int kvm_vgic_get_max_vcpus(void)
+{
+	return kvm_vgic_global_state.max_gic_vcpus;
+}
+
+#endif /* __KVM_ARM_VGIC_H */