From e9b4e73728c78a45516aec27b07bb817ac10eaff Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 18:11:02 +0000 Subject: [PATCH 01/16] irq-apple-aic: minor fixes Obvious minor bugs; neither of them should ordinarily have been an issue for anything but readability. Signed-off-by: Pip Cet --- drivers/irqchip/irq-apple-aic.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index b8c06bd8659e91..c9e2d1105cda1e 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -216,7 +216,7 @@ static void aic_irq_unmask(struct irq_data *d) { struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - aic_ic_write(ic, AIC_MASK_CLR + MASK_REG(d->hwirq), + aic_ic_write(ic, AIC_MASK_CLR + MASK_REG(irqd_to_hwirq(d)), MASK_BIT(irqd_to_hwirq(d))); } @@ -723,7 +723,7 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI, NUMA_NO_NODE, NULL, false, NULL); - if (WARN_ON(!base_ipi)) { + if (WARN_ON(base_ipi < 0)) { irq_domain_remove(ipi_domain); return -ENODEV; } From 42c20f61387ca0c1d2bb725a22e413f7877be6d1 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 18:11:32 +0000 Subject: [PATCH 02/16] irq-apple-aic: don't keep unmasked "disabled" IRQs masked Bug found by Sven Peter. The problem here is that when lazy IRQ disabling is in effect, an IRQ may be logically disabled but physically unmasked. Failing to unmask such an IRQ after masking it will confuse the generic IRQ layer. Signed-off-by: Pip Cet --- drivers/irqchip/irq-apple-aic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index c9e2d1105cda1e..5c11b775f60766 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -226,7 +226,7 @@ static void aic_irq_eoi(struct irq_data *d) * Reading the interrupt reason automatically acknowledges and masks * the IRQ, so we just unmask it here if needed. */ - if (!irqd_irq_disabled(d) && !irqd_irq_masked(d)) + if (!irqd_irq_masked(d)) aic_irq_unmask(d); } From bff3b7e74a5242d4cf748214e5a114f9bdaf27e3 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 18:37:10 +0000 Subject: [PATCH 03/16] irq-apple-aic: Add multi-CPU affinities The AIC on the Apple M1 SoC does support sending interrupts to whichever CPU is "available" (speculation is that it does so by inspecting the CPUs' PSTATE and choosing the lowest-numbered CPU which is not masking interrupts). Let's make use of it. Signed-off-by: Pip Cet --- drivers/irqchip/irq-apple-aic.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 5c11b775f60766..4d846a2b6a35a1 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -271,14 +271,13 @@ static int aic_irq_set_affinity(struct irq_data *d, irq_hw_number_t hwirq = irqd_to_hwirq(d); struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); int cpu; + u32 mask = 0; - if (force) - cpu = cpumask_first(mask_val); - else - cpu = cpumask_any_and(mask_val, cpu_online_mask); + for_each_cpu(cpu, mask_val) + mask |= BIT(cpu); - aic_ic_write(ic, AIC_TARGET_CPU + hwirq * 4, BIT(cpu)); - irq_data_update_effective_affinity(d, cpumask_of(cpu)); + aic_ic_write(ic, AIC_TARGET_CPU + hwirq * 4, mask); + irq_data_update_effective_affinity(d, mask_val); return IRQ_SET_MASK_OK; } From a6a7537d6770a767d892c3c629f31feaadf1d13b Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 18:46:05 +0000 Subject: [PATCH 04/16] irq-apple-aic: prepare for forwarding both IPIs This changes the API to pass an IPI index. Signed-off-by: Pip Cet --- drivers/irqchip/irq-apple-aic.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 4d846a2b6a35a1..41bf9d79031b59 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -245,9 +245,9 @@ static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) irq = FIELD_GET(AIC_EVENT_NUM, event); if (type == AIC_EVENT_TYPE_HW) - handle_domain_irq(aic_irqc->hw_domain, irq, regs); - else if (type == AIC_EVENT_TYPE_IPI && irq == 1) - aic_handle_ipi(regs); + handle_domain_irq(ic->hw_domain, irq, regs); + else if (type == AIC_EVENT_TYPE_IPI) + aic_handle_ipi(0 /* irq */, regs); else if (event != 0) pr_err_ratelimited("Unknown IRQ event %d, %d\n", type, irq); } while (event); From 39f34a418e210ce347791843524536d8d7fc26cf Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 16:16:36 +0000 Subject: [PATCH 05/16] arm64: vipi.c: copy irq-apple-aic.c No further changes. Signed-off-by: Pip Cet --- arch/arm64/kernel/vipi.c | 860 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 860 insertions(+) create mode 100644 arch/arm64/kernel/vipi.c diff --git a/arch/arm64/kernel/vipi.c b/arch/arm64/kernel/vipi.c new file mode 100644 index 00000000000000..41bf9d79031b59 --- /dev/null +++ b/arch/arm64/kernel/vipi.c @@ -0,0 +1,860 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright The Asahi Linux Contributors + * + * Based on irq-lpc32xx: + * Copyright 2015-2016 Vladimir Zapolskiy + * Based on irq-bcm2836: + * Copyright 2015 Broadcom + */ + +/* + * AIC is a fairly simple interrupt controller with the following features: + * + * - 896 level-triggered hardware IRQs + * - Single mask bit per IRQ + * - Per-IRQ affinity setting + * - Automatic masking on event delivery (auto-ack) + * - Software triggering (ORed with hw line) + * - 2 per-CPU IPIs (meant as "self" and "other", but they are + * interchangeable if not symmetric) + * - Automatic prioritization (single event/ack register per CPU, lower IRQs = + * higher priority) + * - Automatic masking on ack + * - Default "this CPU" register view and explicit per-CPU views + * + * In addition, this driver also handles FIQs, as these are routed to the same + * IRQ vector. These are used for Fast IPIs (TODO), the ARMv8 timer IRQs, and + * performance counters (TODO). + * + * Implementation notes: + * + * - This driver creates two IRQ domains, one for HW IRQs and internal FIQs, + * and one for IPIs. + * - Since Linux needs more than 2 IPIs, we implement a software IRQ controller + * and funnel all IPIs into one per-CPU IPI (the second "self" IPI is unused). + * - FIQ hwirq numbers are assigned after true hwirqs, and are per-cpu. + * - DT bindings use 3-cell form (like GIC): + * - <0 nr flags> - hwirq #nr + * - <1 nr flags> - FIQ #nr + * - nr=0 Physical HV timer + * - nr=1 Virtual HV timer + * - nr=2 Physical guest timer + * - nr=3 Virtual guest timer + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* + * AIC registers (MMIO) + */ + +#define AIC_INFO 0x0004 +#define AIC_INFO_NR_HW GENMASK(15, 0) + +#define AIC_CONFIG 0x0010 + +#define AIC_WHOAMI 0x2000 +#define AIC_EVENT 0x2004 +#define AIC_EVENT_TYPE GENMASK(31, 16) +#define AIC_EVENT_NUM GENMASK(15, 0) + +#define AIC_EVENT_TYPE_HW 1 +#define AIC_EVENT_TYPE_IPI 4 +#define AIC_EVENT_IPI_OTHER 1 +#define AIC_EVENT_IPI_SELF 2 + +#define AIC_IPI_SEND 0x2008 +#define AIC_IPI_ACK 0x200c +#define AIC_IPI_MASK_SET 0x2024 +#define AIC_IPI_MASK_CLR 0x2028 + +#define AIC_IPI_SEND_CPU(cpu) BIT(cpu) + +#define AIC_IPI_OTHER BIT(0) +#define AIC_IPI_SELF BIT(31) + +#define AIC_TARGET_CPU 0x3000 +#define AIC_SW_SET 0x4000 +#define AIC_SW_CLR 0x4080 +#define AIC_MASK_SET 0x4100 +#define AIC_MASK_CLR 0x4180 + +#define AIC_CPU_IPI_SET(cpu) (0x5008 + ((cpu) << 7)) +#define AIC_CPU_IPI_CLR(cpu) (0x500c + ((cpu) << 7)) +#define AIC_CPU_IPI_MASK_SET(cpu) (0x5024 + ((cpu) << 7)) +#define AIC_CPU_IPI_MASK_CLR(cpu) (0x5028 + ((cpu) << 7)) + +#define MASK_REG(x) (4 * ((x) >> 5)) +#define MASK_BIT(x) BIT((x) & GENMASK(4, 0)) + +/* + * IMP-DEF sysregs that control FIQ sources + * Note: sysreg-based IPIs are not supported yet. + */ + +/* Core PMC control register */ +#define SYS_IMP_APL_PMCR0_EL1 sys_reg(3, 1, 15, 0, 0) +#define PMCR0_IMODE GENMASK(10, 8) +#define PMCR0_IMODE_OFF 0 +#define PMCR0_IMODE_PMI 1 +#define PMCR0_IMODE_AIC 2 +#define PMCR0_IMODE_HALT 3 +#define PMCR0_IMODE_FIQ 4 +#define PMCR0_IACT BIT(11) + +/* IPI request registers */ +#define SYS_IMP_APL_IPI_RR_LOCAL_EL1 sys_reg(3, 5, 15, 0, 0) +#define SYS_IMP_APL_IPI_RR_GLOBAL_EL1 sys_reg(3, 5, 15, 0, 1) +#define IPI_RR_CPU GENMASK(7, 0) +/* Cluster only used for the GLOBAL register */ +#define IPI_RR_CLUSTER GENMASK(23, 16) +#define IPI_RR_TYPE GENMASK(29, 28) +#define IPI_RR_IMMEDIATE 0 +#define IPI_RR_RETRACT 1 +#define IPI_RR_DEFERRED 2 +#define IPI_RR_NOWAKE 3 + +/* IPI status register */ +#define SYS_IMP_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) +#define IPI_SR_PENDING BIT(0) + +/* Guest timer FIQ enable register */ +#define SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2 sys_reg(3, 5, 15, 1, 3) +#define VM_TMR_FIQ_ENABLE_V BIT(0) +#define VM_TMR_FIQ_ENABLE_P BIT(1) + +/* Deferred IPI countdown register */ +#define SYS_IMP_APL_IPI_CR_EL1 sys_reg(3, 5, 15, 3, 1) + +/* Uncore PMC control register */ +#define SYS_IMP_APL_UPMCR0_EL1 sys_reg(3, 7, 15, 0, 4) +#define UPMCR0_IMODE GENMASK(18, 16) +#define UPMCR0_IMODE_OFF 0 +#define UPMCR0_IMODE_AIC 2 +#define UPMCR0_IMODE_HALT 3 +#define UPMCR0_IMODE_FIQ 4 + +/* Uncore PMC status register */ +#define SYS_IMP_APL_UPMSR_EL1 sys_reg(3, 7, 15, 6, 4) +#define UPMSR_IACT BIT(0) + +#define AIC_NR_FIQ 4 +#define AIC_NR_SWIPI 32 + +/* + * FIQ hwirq index definitions: FIQ sources use the DT binding defines + * directly, except that timers are special. At the irqchip level, the + * two timer types are represented by their access method: _EL0 registers + * or _EL02 registers. In the DT binding, the timers are represented + * by their purpose (HV or guest). This mapping is for when the kernel is + * running at EL2 (with VHE). When the kernel is running at EL1, the + * mapping differs and aic_irq_domain_translate() performs the remapping. + */ + +#define AIC_TMR_EL0_PHYS AIC_TMR_HV_PHYS +#define AIC_TMR_EL0_VIRT AIC_TMR_HV_VIRT +#define AIC_TMR_EL02_PHYS AIC_TMR_GUEST_PHYS +#define AIC_TMR_EL02_VIRT AIC_TMR_GUEST_VIRT + +struct aic_irq_chip { + void __iomem *base; + struct irq_domain *hw_domain; + struct irq_domain *ipi_domain; + int nr_hw; + int ipi_hwirq; +}; + +static DEFINE_PER_CPU(uint32_t, aic_fiq_unmasked); + +static DEFINE_PER_CPU(atomic_t, aic_vipi_flag); +static DEFINE_PER_CPU(atomic_t, aic_vipi_enable); + +static struct aic_irq_chip *aic_irqc; + +static void aic_handle_ipi(struct pt_regs *regs); + +static u32 aic_ic_read(struct aic_irq_chip *ic, u32 reg) +{ + return readl_relaxed(ic->base + reg); +} + +static void aic_ic_write(struct aic_irq_chip *ic, u32 reg, u32 val) +{ + writel_relaxed(val, ic->base + reg); +} + +/* + * IRQ irqchip + */ + +static void aic_irq_mask(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + + aic_ic_write(ic, AIC_MASK_SET + MASK_REG(irqd_to_hwirq(d)), + MASK_BIT(irqd_to_hwirq(d))); +} + +static void aic_irq_unmask(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + + aic_ic_write(ic, AIC_MASK_CLR + MASK_REG(irqd_to_hwirq(d)), + MASK_BIT(irqd_to_hwirq(d))); +} + +static void aic_irq_eoi(struct irq_data *d) +{ + /* + * Reading the interrupt reason automatically acknowledges and masks + * the IRQ, so we just unmask it here if needed. + */ + if (!irqd_irq_masked(d)) + aic_irq_unmask(d); +} + +static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 event, type, irq; + + do { + /* + * We cannot use a relaxed read here, as reads from DMA buffers + * need to be ordered after the IRQ fires. + */ + event = readl(ic->base + AIC_EVENT); + type = FIELD_GET(AIC_EVENT_TYPE, event); + irq = FIELD_GET(AIC_EVENT_NUM, event); + + if (type == AIC_EVENT_TYPE_HW) + handle_domain_irq(ic->hw_domain, irq, regs); + else if (type == AIC_EVENT_TYPE_IPI) + aic_handle_ipi(0 /* irq */, regs); + else if (event != 0) + pr_err_ratelimited("Unknown IRQ event %d, %d\n", type, irq); + } while (event); + + /* + * vGIC maintenance interrupts end up here too, so we need to check + * for them separately. This should never trigger if KVM is working + * properly, because it will have already taken care of clearing it + * on guest exit before this handler runs. + */ + if (is_kernel_in_hyp_mode() && (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EN) && + read_sysreg_s(SYS_ICH_MISR_EL2) != 0) { + pr_err_ratelimited("vGIC IRQ fired and not handled by KVM, disabling.\n"); + sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); + } +} + +static int aic_irq_set_affinity(struct irq_data *d, + const struct cpumask *mask_val, bool force) +{ + irq_hw_number_t hwirq = irqd_to_hwirq(d); + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + int cpu; + u32 mask = 0; + + for_each_cpu(cpu, mask_val) + mask |= BIT(cpu); + + aic_ic_write(ic, AIC_TARGET_CPU + hwirq * 4, mask); + irq_data_update_effective_affinity(d, mask_val); + + return IRQ_SET_MASK_OK; +} + +static int aic_irq_set_type(struct irq_data *d, unsigned int type) +{ + /* + * Some IRQs (e.g. MSIs) implicitly have edge semantics, and we don't + * have a way to find out the type of any given IRQ, so just allow both. + */ + return (type == IRQ_TYPE_LEVEL_HIGH || type == IRQ_TYPE_EDGE_RISING) ? 0 : -EINVAL; +} + +static struct irq_chip aic_chip = { + .name = "AIC", + .irq_mask = aic_irq_mask, + .irq_unmask = aic_irq_unmask, + .irq_eoi = aic_irq_eoi, + .irq_set_affinity = aic_irq_set_affinity, + .irq_set_type = aic_irq_set_type, +}; + +/* + * FIQ irqchip + */ + +static unsigned long aic_fiq_get_idx(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + + return irqd_to_hwirq(d) - ic->nr_hw; +} + +static void aic_fiq_set_mask(struct irq_data *d) +{ + /* Only the guest timers have real mask bits, unfortunately. */ + switch (aic_fiq_get_idx(d)) { + case AIC_TMR_EL02_PHYS: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_P, 0); + isb(); + break; + case AIC_TMR_EL02_VIRT: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_V, 0); + isb(); + break; + default: + break; + } +} + +static void aic_fiq_clear_mask(struct irq_data *d) +{ + switch (aic_fiq_get_idx(d)) { + case AIC_TMR_EL02_PHYS: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_P); + isb(); + break; + case AIC_TMR_EL02_VIRT: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_V); + isb(); + break; + default: + break; + } +} + +static void aic_fiq_mask(struct irq_data *d) +{ + aic_fiq_set_mask(d); + __this_cpu_and(aic_fiq_unmasked, ~BIT(aic_fiq_get_idx(d))); +} + +static void aic_fiq_unmask(struct irq_data *d) +{ + aic_fiq_clear_mask(d); + __this_cpu_or(aic_fiq_unmasked, BIT(aic_fiq_get_idx(d))); +} + +static void aic_fiq_eoi(struct irq_data *d) +{ + /* We mask to ack (where we can), so we need to unmask at EOI. */ + if (__this_cpu_read(aic_fiq_unmasked) & BIT(aic_fiq_get_idx(d))) + aic_fiq_clear_mask(d); +} + +#define TIMER_FIRING(x) \ + (((x) & (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_MASK | \ + ARCH_TIMER_CTRL_IT_STAT)) == \ + (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_STAT)) + +static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) +{ + /* + * It would be really nice if we had a system register that lets us get + * the FIQ source state without having to peek down into sources... + * but such a register does not seem to exist. + * + * So, we have these potential sources to test for: + * - Fast IPIs (not yet used) + * - The 4 timers (CNTP, CNTV for each of HV and guest) + * - Per-core PMCs (not yet supported) + * - Per-cluster uncore PMCs (not yet supported) + * + * Since not dealing with any of these results in a FIQ storm, + * we check for everything here, even things we don't support yet. + */ + + if (read_sysreg_s(SYS_IMP_APL_IPI_SR_EL1) & IPI_SR_PENDING) { + pr_err_ratelimited("Fast IPI fired. Acking.\n"); + write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); + } + + if (TIMER_FIRING(read_sysreg(cntp_ctl_el0))) + handle_domain_irq(aic_irqc->hw_domain, + aic_irqc->nr_hw + AIC_TMR_EL0_PHYS, regs); + + if (TIMER_FIRING(read_sysreg(cntv_ctl_el0))) + handle_domain_irq(aic_irqc->hw_domain, + aic_irqc->nr_hw + AIC_TMR_EL0_VIRT, regs); + + if (is_kernel_in_hyp_mode()) { + uint64_t enabled = read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2); + + if ((enabled & VM_TMR_FIQ_ENABLE_P) && + TIMER_FIRING(read_sysreg_s(SYS_CNTP_CTL_EL02))) + handle_domain_irq(aic_irqc->hw_domain, + aic_irqc->nr_hw + AIC_TMR_EL02_PHYS, regs); + + if ((enabled & VM_TMR_FIQ_ENABLE_V) && + TIMER_FIRING(read_sysreg_s(SYS_CNTV_CTL_EL02))) + handle_domain_irq(aic_irqc->hw_domain, + aic_irqc->nr_hw + AIC_TMR_EL02_VIRT, regs); + } + + if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == + (FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_FIQ) | PMCR0_IACT)) { + /* + * Not supported yet, let's figure out how to handle this when + * we implement these proprietary performance counters. For now, + * just mask it and move on. + */ + pr_err_ratelimited("PMC FIQ fired. Masking.\n"); + sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, + FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); + } + + if (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ && + (read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) { + /* Same story with uncore PMCs */ + pr_err_ratelimited("Uncore PMC FIQ fired. Masking.\n"); + sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, + FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); + } +} + +static int aic_fiq_set_type(struct irq_data *d, unsigned int type) +{ + return (type == IRQ_TYPE_LEVEL_HIGH) ? 0 : -EINVAL; +} + +static struct irq_chip fiq_chip = { + .name = "AIC-FIQ", + .irq_mask = aic_fiq_mask, + .irq_unmask = aic_fiq_unmask, + .irq_ack = aic_fiq_set_mask, + .irq_eoi = aic_fiq_eoi, + .irq_set_type = aic_fiq_set_type, +}; + +/* + * Main IRQ domain + */ + +static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, + irq_hw_number_t hw) +{ + struct aic_irq_chip *ic = id->host_data; + + if (hw < ic->nr_hw) { + irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, + handle_fasteoi_irq, NULL, NULL); + irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); + } else { + irq_set_percpu_devid(irq); + irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, + handle_percpu_devid_irq, NULL, NULL); + } + + return 0; +} + +static int aic_irq_domain_translate(struct irq_domain *id, + struct irq_fwspec *fwspec, + unsigned long *hwirq, + unsigned int *type) +{ + struct aic_irq_chip *ic = id->host_data; + + if (fwspec->param_count != 3 || !is_of_node(fwspec->fwnode)) + return -EINVAL; + + switch (fwspec->param[0]) { + case AIC_IRQ: + if (fwspec->param[1] >= ic->nr_hw) + return -EINVAL; + *hwirq = fwspec->param[1]; + break; + case AIC_FIQ: + if (fwspec->param[1] >= AIC_NR_FIQ) + return -EINVAL; + *hwirq = ic->nr_hw + fwspec->param[1]; + + /* + * In EL1 the non-redirected registers are the guest's, + * not EL2's, so remap the hwirqs to match. + */ + if (!is_kernel_in_hyp_mode()) { + switch (fwspec->param[1]) { + case AIC_TMR_GUEST_PHYS: + *hwirq = ic->nr_hw + AIC_TMR_EL0_PHYS; + break; + case AIC_TMR_GUEST_VIRT: + *hwirq = ic->nr_hw + AIC_TMR_EL0_VIRT; + break; + case AIC_TMR_HV_PHYS: + case AIC_TMR_HV_VIRT: + return -ENOENT; + default: + break; + } + } + break; + default: + return -EINVAL; + } + + *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; + + return 0; +} + +static int aic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + unsigned int type = IRQ_TYPE_NONE; + struct irq_fwspec *fwspec = arg; + irq_hw_number_t hwirq; + int i, ret; + + ret = aic_irq_domain_translate(domain, fwspec, &hwirq, &type); + if (ret) + return ret; + + for (i = 0; i < nr_irqs; i++) { + ret = aic_irq_domain_map(domain, virq + i, hwirq + i); + if (ret) + return ret; + } + + return 0; +} + +static void aic_irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } +} + +static const struct irq_domain_ops aic_irq_domain_ops = { + .translate = aic_irq_domain_translate, + .alloc = aic_irq_domain_alloc, + .free = aic_irq_domain_free, +}; + +/* + * IPI irqchip + */ + +static void aic_ipi_mask(struct irq_data *d) +{ + u32 irq_bit = BIT(irqd_to_hwirq(d)); + + /* No specific ordering requirements needed here. */ + atomic_andnot(irq_bit, this_cpu_ptr(&aic_vipi_enable)); +} + +static void aic_ipi_unmask(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + u32 irq_bit = BIT(irqd_to_hwirq(d)); + + atomic_or(irq_bit, this_cpu_ptr(&aic_vipi_enable)); + + /* + * The atomic_or() above must complete before the atomic_read() + * below to avoid racing aic_ipi_send_mask(). + */ + smp_mb__after_atomic(); + + /* + * If a pending vIPI was unmasked, raise a HW IPI to ourselves. + * No barriers needed here since this is a self-IPI. + */ + if (atomic_read(this_cpu_ptr(&aic_vipi_flag)) & irq_bit) + aic_ic_write(ic, AIC_IPI_SEND, AIC_IPI_SEND_CPU(smp_processor_id())); +} + +static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + u32 irq_bit = BIT(irqd_to_hwirq(d)); + u32 send = 0; + int cpu; + unsigned long pending; + + for_each_cpu(cpu, mask) { + /* + * This sequence is the mirror of the one in aic_ipi_unmask(); + * see the comment there. Additionally, release semantics + * ensure that the vIPI flag set is ordered after any shared + * memory accesses that precede it. This therefore also pairs + * with the atomic_fetch_andnot in aic_handle_ipi(). + */ + pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&aic_vipi_flag, cpu)); + + /* + * The atomic_fetch_or_release() above must complete before the + * atomic_read() below to avoid racing aic_ipi_unmask(). + */ + smp_mb__after_atomic(); + + if (!(pending & irq_bit) && + (atomic_read(per_cpu_ptr(&aic_vipi_enable, cpu)) & irq_bit)) + send |= AIC_IPI_SEND_CPU(cpu); + } + + /* + * The flag writes must complete before the physical IPI is issued + * to another CPU. This is implied by the control dependency on + * the result of atomic_read_acquire() above, which is itself + * already ordered after the vIPI flag write. + */ + if (send) + aic_ic_write(ic, AIC_IPI_SEND, send); +} + +static struct irq_chip ipi_chip = { + .name = "AIC-IPI", + .irq_mask = aic_ipi_mask, + .irq_unmask = aic_ipi_unmask, + .ipi_send_mask = aic_ipi_send_mask, +}; + +/* + * IPI IRQ domain + */ + +static void aic_handle_ipi(struct pt_regs *regs) +{ + int i; + unsigned long enabled, firing; + + /* + * Ack the IPI. We need to order this after the AIC event read, but + * that is enforced by normal MMIO ordering guarantees. + */ + aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_OTHER); + + /* + * The mask read does not need to be ordered. Only we can change + * our own mask anyway, so no races are possible here, as long as + * we are properly in the interrupt handler (which is covered by + * the barrier that is part of the top-level AIC handler's readl()). + */ + enabled = atomic_read(this_cpu_ptr(&aic_vipi_enable)); + + /* + * Clear the IPIs we are about to handle. This pairs with the + * atomic_fetch_or_release() in aic_ipi_send_mask(), and needs to be + * ordered after the aic_ic_write() above (to avoid dropping vIPIs) and + * before IPI handling code (to avoid races handling vIPIs before they + * are signaled). The former is taken care of by the release semantics + * of the write portion, while the latter is taken care of by the + * acquire semantics of the read portion. + */ + firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&aic_vipi_flag)) & enabled; + + for_each_set_bit(i, &firing, AIC_NR_SWIPI) + handle_domain_irq(aic_irqc->ipi_domain, i, regs); + + /* + * No ordering needed here; at worst this just changes the timing of + * when the next IPI will be delivered. + */ + aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); +} + +static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + irq_set_percpu_devid(virq + i); + irq_domain_set_info(d, virq + i, i, &ipi_chip, d->host_data, + handle_percpu_devid_irq, NULL, NULL); + } + + return 0; +} + +static void aic_ipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) +{ + /* Not freeing IPIs */ +} + +static const struct irq_domain_ops aic_ipi_domain_ops = { + .alloc = aic_ipi_alloc, + .free = aic_ipi_free, +}; + +static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) +{ + struct irq_domain *ipi_domain; + int base_ipi; + + ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_SWIPI, + &aic_ipi_domain_ops, irqc); + if (WARN_ON(!ipi_domain)) + return -ENODEV; + + ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; + irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); + + base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI, + NUMA_NO_NODE, NULL, false, NULL); + + if (WARN_ON(base_ipi < 0)) { + irq_domain_remove(ipi_domain); + return -ENODEV; + } + + set_smp_ipi_range(base_ipi, AIC_NR_SWIPI); + + irqc->ipi_domain = ipi_domain; + + return 0; +} + +static int aic_init_cpu(unsigned int cpu) +{ + /* Mask all hard-wired per-CPU IRQ/FIQ sources */ + + /* Pending Fast IPI FIQs */ + write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); + + /* Timer FIQs */ + sysreg_clear_set(cntp_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); + sysreg_clear_set(cntv_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); + + /* EL2-only (VHE mode) IRQ sources */ + if (is_kernel_in_hyp_mode()) { + /* Guest timers */ + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, + VM_TMR_FIQ_ENABLE_V | VM_TMR_FIQ_ENABLE_P, 0); + + /* vGIC maintenance IRQ */ + sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); + } + + /* PMC FIQ */ + sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, + FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); + + /* Uncore PMC FIQ */ + sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, + FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); + + /* Commit all of the above */ + isb(); + + /* + * Make sure the kernel's idea of logical CPU order is the same as AIC's + * If we ever end up with a mismatch here, we will have to introduce + * a mapping table similar to what other irqchip drivers do. + */ + WARN_ON(aic_ic_read(aic_irqc, AIC_WHOAMI) != smp_processor_id()); + + /* + * Always keep IPIs unmasked at the hardware level (except auto-masking + * by AIC during processing). We manage masks at the vIPI level. + */ + aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_SELF | AIC_IPI_OTHER); + aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_SELF); + aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); + + /* Initialize the local mask state */ + __this_cpu_write(aic_fiq_unmasked, 0); + + return 0; +} + +static struct gic_kvm_info vgic_info __initdata = { + .type = GIC_V3, + .no_maint_irq_mask = true, + .no_hw_deactivation = true, +}; + +static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) +{ + int i; + void __iomem *regs; + u32 info; + struct aic_irq_chip *irqc; + + regs = of_iomap(node, 0); + if (WARN_ON(!regs)) + return -EIO; + + irqc = kzalloc(sizeof(*irqc), GFP_KERNEL); + if (!irqc) + return -ENOMEM; + + aic_irqc = irqc; + irqc->base = regs; + + info = aic_ic_read(irqc, AIC_INFO); + irqc->nr_hw = FIELD_GET(AIC_INFO_NR_HW, info); + + irqc->hw_domain = irq_domain_create_linear(of_node_to_fwnode(node), + irqc->nr_hw + AIC_NR_FIQ, + &aic_irq_domain_ops, irqc); + if (WARN_ON(!irqc->hw_domain)) { + iounmap(irqc->base); + kfree(irqc); + return -ENODEV; + } + + irq_domain_update_bus_token(irqc->hw_domain, DOMAIN_BUS_WIRED); + + if (aic_init_smp(irqc, node)) { + irq_domain_remove(irqc->hw_domain); + iounmap(irqc->base); + kfree(irqc); + return -ENODEV; + } + + set_handle_irq(aic_handle_irq); + set_handle_fiq(aic_handle_fiq); + + for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) + aic_ic_write(irqc, AIC_MASK_SET + i * 4, U32_MAX); + for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) + aic_ic_write(irqc, AIC_SW_CLR + i * 4, U32_MAX); + for (i = 0; i < irqc->nr_hw; i++) + aic_ic_write(irqc, AIC_TARGET_CPU + i * 4, 1); + + if (!is_kernel_in_hyp_mode()) + pr_info("Kernel running in EL1, mapping interrupts"); + + cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_AIC_STARTING, + "irqchip/apple-aic/ipi:starting", + aic_init_cpu, NULL); + + vgic_set_kvm_info(&vgic_info); + + pr_info("Initialized with %d IRQs, %d FIQs, %d vIPIs\n", + irqc->nr_hw, AIC_NR_FIQ, AIC_NR_SWIPI); + + return 0; +} + +IRQCHIP_DECLARE(apple_m1_aic, "apple,aic", aic_of_ic_init); From 636a404de82303ebd314b22de4bc90558a20aa51 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 16:17:13 +0000 Subject: [PATCH 06/16] irq-apple-fiq: copy irq-apple-aic.c No further changes. Signed-off-by: Pip Cet --- drivers/irqchip/irq-apple-fiq.c | 875 ++++++++++++++++++++++++++++++++ 1 file changed, 875 insertions(+) create mode 100644 drivers/irqchip/irq-apple-fiq.c diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c new file mode 100644 index 00000000000000..13900b4a2ca56e --- /dev/null +++ b/drivers/irqchip/irq-apple-fiq.c @@ -0,0 +1,875 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright The Asahi Linux Contributors + * + * Based on irq-lpc32xx: + * Copyright 2015-2016 Vladimir Zapolskiy + * Based on irq-bcm2836: + * Copyright 2015 Broadcom + */ + +/* + * AIC is a fairly simple interrupt controller with the following features: + * + * - 896 level-triggered hardware IRQs + * - Single mask bit per IRQ + * - Per-IRQ affinity setting + * - Automatic masking on event delivery (auto-ack) + * - Software triggering (ORed with hw line) + * - 2 per-CPU IPIs (meant as "self" and "other", but they are + * interchangeable if not symmetric) + * - Automatic prioritization (single event/ack register per CPU, lower IRQs = + * higher priority) + * - Automatic masking on ack + * - Default "this CPU" register view and explicit per-CPU views + * + * In addition, this driver also handles FIQs, as these are routed to the same + * IRQ vector. These are used for Fast IPIs (TODO), the ARMv8 timer IRQs, and + * performance counters (TODO). + * + * Implementation notes: + * + * - This driver creates two IRQ domains, one for HW IRQs and internal FIQs, + * and one for IPIs. + * - Since Linux needs more than 2 IPIs, we implement a software IRQ controller + * and funnel all IPIs into one per-CPU IPI (the second "self" IPI is unused). + * - FIQ hwirq numbers are assigned after true hwirqs, and are per-cpu. + * - DT bindings use 3-cell form (like GIC): + * - <0 nr flags> - hwirq #nr + * - <1 nr flags> - FIQ #nr + * - nr=0 Physical HV timer + * - nr=1 Virtual HV timer + * - nr=2 Physical guest timer + * - nr=3 Virtual guest timer + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* + * AIC registers (MMIO) + */ + +#define AIC_INFO 0x0004 +#define AIC_INFO_NR_HW GENMASK(15, 0) + +#define AIC_CONFIG 0x0010 + +#define AIC_WHOAMI 0x2000 +#define AIC_EVENT 0x2004 +#define AIC_EVENT_TYPE GENMASK(31, 16) +#define AIC_EVENT_NUM GENMASK(15, 0) + +#define AIC_EVENT_TYPE_HW 1 +#define AIC_EVENT_TYPE_IPI 4 +#define AIC_EVENT_IPI_OTHER 1 +#define AIC_EVENT_IPI_SELF 2 + +#define AIC_IPI_SEND 0x2008 +#define AIC_IPI_ACK 0x200c +#define AIC_IPI_MASK_SET 0x2024 +#define AIC_IPI_MASK_CLR 0x2028 + +#define AIC_IPI_SEND_CPU(cpu) BIT(cpu) + +#define AIC_IPI_OTHER BIT(0) +#define AIC_IPI_SELF BIT(31) + +#define AIC_TARGET_CPU 0x3000 +#define AIC_SW_SET 0x4000 +#define AIC_SW_CLR 0x4080 +#define AIC_MASK_SET 0x4100 +#define AIC_MASK_CLR 0x4180 + +#define AIC_CPU_IPI_SET(cpu) (0x5008 + ((cpu) << 7)) +#define AIC_CPU_IPI_CLR(cpu) (0x500c + ((cpu) << 7)) +#define AIC_CPU_IPI_MASK_SET(cpu) (0x5024 + ((cpu) << 7)) +#define AIC_CPU_IPI_MASK_CLR(cpu) (0x5028 + ((cpu) << 7)) + +#define MASK_REG(x) (4 * ((x) >> 5)) +#define MASK_BIT(x) BIT((x) & GENMASK(4, 0)) + +/* + * IMP-DEF sysregs that control FIQ sources + * Note: sysreg-based IPIs are not supported yet. + */ + +/* Core PMC control register */ +#define SYS_IMP_APL_PMCR0_EL1 sys_reg(3, 1, 15, 0, 0) +#define PMCR0_IMODE GENMASK(10, 8) +#define PMCR0_IMODE_OFF 0 +#define PMCR0_IMODE_PMI 1 +#define PMCR0_IMODE_AIC 2 +#define PMCR0_IMODE_HALT 3 +#define PMCR0_IMODE_FIQ 4 +#define PMCR0_IACT BIT(11) + +/* IPI request registers */ +#define SYS_IMP_APL_IPI_RR_LOCAL_EL1 sys_reg(3, 5, 15, 0, 0) +#define SYS_IMP_APL_IPI_RR_GLOBAL_EL1 sys_reg(3, 5, 15, 0, 1) +#define IPI_RR_CPU GENMASK(7, 0) +/* Cluster only used for the GLOBAL register */ +#define IPI_RR_CLUSTER GENMASK(23, 16) +#define IPI_RR_TYPE GENMASK(29, 28) +#define IPI_RR_IMMEDIATE 0 +#define IPI_RR_RETRACT 1 +#define IPI_RR_DEFERRED 2 +#define IPI_RR_NOWAKE 3 + +/* IPI status register */ +#define SYS_IMP_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) +#define IPI_SR_PENDING BIT(0) + +/* Guest timer FIQ enable register */ +#define SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2 sys_reg(3, 5, 15, 1, 3) +#define VM_TMR_FIQ_ENABLE_V BIT(0) +#define VM_TMR_FIQ_ENABLE_P BIT(1) + +/* Deferred IPI countdown register */ +#define SYS_IMP_APL_IPI_CR_EL1 sys_reg(3, 5, 15, 3, 1) + +/* Uncore PMC control register */ +#define SYS_IMP_APL_UPMCR0_EL1 sys_reg(3, 7, 15, 0, 4) +#define UPMCR0_IMODE GENMASK(18, 16) +#define UPMCR0_IMODE_OFF 0 +#define UPMCR0_IMODE_AIC 2 +#define UPMCR0_IMODE_HALT 3 +#define UPMCR0_IMODE_FIQ 4 + +/* Uncore PMC status register */ +#define SYS_IMP_APL_UPMSR_EL1 sys_reg(3, 7, 15, 6, 4) +#define UPMSR_IACT BIT(0) + +#define AIC_NR_FIQ 4 +#define AIC_NR_SWIPI 32 + +/* + * FIQ hwirq index definitions: FIQ sources use the DT binding defines + * directly, except that timers are special. At the irqchip level, the + * two timer types are represented by their access method: _EL0 registers + * or _EL02 registers. In the DT binding, the timers are represented + * by their purpose (HV or guest). This mapping is for when the kernel is + * running at EL2 (with VHE). When the kernel is running at EL1, the + * mapping differs and aic_irq_domain_translate() performs the remapping. + */ + +#define AIC_TMR_EL0_PHYS AIC_TMR_HV_PHYS +#define AIC_TMR_EL0_VIRT AIC_TMR_HV_VIRT +#define AIC_TMR_EL02_PHYS AIC_TMR_GUEST_PHYS +#define AIC_TMR_EL02_VIRT AIC_TMR_GUEST_VIRT + +struct aic_irq_chip { + void __iomem *base; + struct irq_domain *hw_domain; + struct irq_domain *ipi_domain; + int nr_hw; + int ipi_hwirq; +}; + +static DEFINE_PER_CPU(uint32_t, aic_fiq_unmasked); + +static DEFINE_PER_CPU(atomic_t, aic_vipi_flag); +static DEFINE_PER_CPU(atomic_t, aic_vipi_enable); + +static struct aic_irq_chip *aic_irqc; + +static void aic_handle_ipi(struct pt_regs *regs); + +static u32 aic_ic_read(struct aic_irq_chip *ic, u32 reg) +{ + return readl_relaxed(ic->base + reg); +} + +static void aic_ic_write(struct aic_irq_chip *ic, u32 reg, u32 val) +{ + writel_relaxed(val, ic->base + reg); +} + +/* + * IRQ irqchip + */ + +static void aic_irq_mask(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + + aic_ic_write(ic, AIC_MASK_SET + MASK_REG(irqd_to_hwirq(d)), + MASK_BIT(irqd_to_hwirq(d))); +} + +static void aic_irq_unmask(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + + aic_ic_write(ic, AIC_MASK_CLR + MASK_REG(irqd_to_hwirq(d)), + MASK_BIT(irqd_to_hwirq(d))); +} + +static void aic_irq_eoi(struct irq_data *d) +{ + /* + * Reading the interrupt reason automatically acknowledges and masks + * the IRQ, so we just unmask it here if needed. + */ + if (!irqd_irq_masked(d)) + aic_irq_unmask(d); +} + +static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 event, type, irq; + + do { + /* + * We cannot use a relaxed read here, as reads from DMA buffers + * need to be ordered after the IRQ fires. + */ + event = readl(ic->base + AIC_EVENT); + type = FIELD_GET(AIC_EVENT_TYPE, event); + irq = FIELD_GET(AIC_EVENT_NUM, event); + + if (type == AIC_EVENT_TYPE_HW) + handle_domain_irq(ic->hw_domain, irq, regs); + else if (type == AIC_EVENT_TYPE_IPI) + aic_handle_ipi(0 /* irq */, regs); + else if (event != 0) + pr_err_ratelimited("Unknown IRQ event %d, %d\n", type, irq); + } while (event); + + /* + * vGIC maintenance interrupts end up here too, so we need to check + * for them separately. This should never trigger if KVM is working + * properly, because it will have already taken care of clearing it + * on guest exit before this handler runs. + */ + if (is_kernel_in_hyp_mode() && (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EN) && + read_sysreg_s(SYS_ICH_MISR_EL2) != 0) { + pr_err_ratelimited("vGIC IRQ fired and not handled by KVM, disabling.\n"); + sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); + } +} + +static int aic_irq_set_affinity(struct irq_data *d, + const struct cpumask *mask_val, bool force) +{ + irq_hw_number_t hwirq = irqd_to_hwirq(d); + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + int cpu; + u32 mask = 0; + + for_each_cpu(cpu, mask_val) + mask |= BIT(cpu); + + aic_ic_write(ic, AIC_TARGET_CPU + hwirq * 4, mask); + irq_data_update_effective_affinity(d, mask_val); + + return IRQ_SET_MASK_OK; +} + +static int aic_irq_set_type(struct irq_data *d, unsigned int type) +{ + /* + * Some IRQs (e.g. MSIs) implicitly have edge semantics, and we don't + * have a way to find out the type of any given IRQ, so just allow both. + */ + return (type == IRQ_TYPE_LEVEL_HIGH || type == IRQ_TYPE_EDGE_RISING) ? 0 : -EINVAL; +} + +static struct irq_chip aic_chip = { + .name = "AIC", + .irq_mask = aic_irq_mask, + .irq_unmask = aic_irq_unmask, + .irq_eoi = aic_irq_eoi, + .irq_set_affinity = aic_irq_set_affinity, + .irq_set_type = aic_irq_set_type, +}; + +/* + * FIQ irqchip + */ + +static unsigned long aic_fiq_get_idx(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + + return irqd_to_hwirq(d) - ic->nr_hw; +} + +static void aic_fiq_set_mask(struct irq_data *d) +{ + /* Only the guest timers have real mask bits, unfortunately. */ + switch (aic_fiq_get_idx(d)) { + case AIC_TMR_EL02_PHYS: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_P, 0); + isb(); + break; + case AIC_TMR_EL02_VIRT: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_V, 0); + isb(); + break; + default: + break; + } +} + +static void aic_fiq_clear_mask(struct irq_data *d) +{ + switch (aic_fiq_get_idx(d)) { + case AIC_TMR_EL02_PHYS: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_P); + isb(); + break; + case AIC_TMR_EL02_VIRT: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_V); + isb(); + break; + default: + break; + } +} + +static void aic_fiq_mask(struct irq_data *d) +{ + aic_fiq_set_mask(d); + __this_cpu_and(aic_fiq_unmasked, ~BIT(aic_fiq_get_idx(d))); +} + +static void aic_fiq_unmask(struct irq_data *d) +{ + aic_fiq_clear_mask(d); + __this_cpu_or(aic_fiq_unmasked, BIT(aic_fiq_get_idx(d))); +} + +static void aic_fiq_eoi(struct irq_data *d) +{ + /* We mask to ack (where we can), so we need to unmask at EOI. */ + if (__this_cpu_read(aic_fiq_unmasked) & BIT(aic_fiq_get_idx(d))) + aic_fiq_clear_mask(d); +} + +#define TIMER_FIRING(x) \ + (((x) & (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_MASK | \ + ARCH_TIMER_CTRL_IT_STAT)) == \ + (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_STAT)) + +static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) +{ + /* + * It would be really nice if we had a system register that lets us get + * the FIQ source state without having to peek down into sources... + * but such a register does not seem to exist. + * + * So, we have these potential sources to test for: + * - Fast IPIs (not yet used) + * - The 4 timers (CNTP, CNTV for each of HV and guest) + * - Per-core PMCs (not yet supported) + * - Per-cluster uncore PMCs (not yet supported) + * + * Since not dealing with any of these results in a FIQ storm, + * we check for everything here, even things we don't support yet. + */ + + if (read_sysreg_s(SYS_IMP_APL_IPI_SR_EL1) & IPI_SR_PENDING) { + pr_err_ratelimited("Fast IPI fired. Acking.\n"); + write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); + } + + if (TIMER_FIRING(read_sysreg(cntp_ctl_el0))) + handle_domain_irq(aic_irqc->hw_domain, + aic_irqc->nr_hw + AIC_TMR_EL0_PHYS, regs); + + if (TIMER_FIRING(read_sysreg(cntv_ctl_el0))) + handle_domain_irq(aic_irqc->hw_domain, + aic_irqc->nr_hw + AIC_TMR_EL0_VIRT, regs); + + if (is_kernel_in_hyp_mode()) { + uint64_t enabled = read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2); + + if ((enabled & VM_TMR_FIQ_ENABLE_P) && + TIMER_FIRING(read_sysreg_s(SYS_CNTP_CTL_EL02))) + handle_domain_irq(aic_irqc->hw_domain, + aic_irqc->nr_hw + AIC_TMR_EL02_PHYS, regs); + + if ((enabled & VM_TMR_FIQ_ENABLE_V) && + TIMER_FIRING(read_sysreg_s(SYS_CNTV_CTL_EL02))) + handle_domain_irq(aic_irqc->hw_domain, + aic_irqc->nr_hw + AIC_TMR_EL02_VIRT, regs); + } + + if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == + (FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_FIQ) | PMCR0_IACT)) { + /* + * Not supported yet, let's figure out how to handle this when + * we implement these proprietary performance counters. For now, + * just mask it and move on. + */ + pr_err_ratelimited("PMC FIQ fired. Masking.\n"); + sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, + FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); + } + + if (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ && + (read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) { + /* Same story with uncore PMCs */ + pr_err_ratelimited("Uncore PMC FIQ fired. Masking.\n"); + sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, + FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); + } +} + +static int aic_fiq_set_type(struct irq_data *d, unsigned int type) +{ + return (type == IRQ_TYPE_LEVEL_HIGH) ? 0 : -EINVAL; +} + +static struct irq_chip fiq_chip = { + .name = "AIC-FIQ", + .irq_mask = aic_fiq_mask, + .irq_unmask = aic_fiq_unmask, + .irq_ack = aic_fiq_set_mask, + .irq_eoi = aic_fiq_eoi, + .irq_set_type = aic_fiq_set_type, +}; + +/* + * Main IRQ domain + */ + +static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, + irq_hw_number_t hw) +{ + struct aic_irq_chip *ic = id->host_data; + + if (hw < ic->nr_hw) { + irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, + handle_fasteoi_irq, NULL, NULL); + irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); + } else { + irq_set_percpu_devid(irq); + irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, + handle_percpu_devid_irq, NULL, NULL); + } + + return 0; +} + +static int aic_irq_domain_translate(struct irq_domain *id, + struct irq_fwspec *fwspec, + unsigned long *hwirq, + unsigned int *type) +{ + struct aic_irq_chip *ic = id->host_data; + + if (fwspec->param_count != 3 || !is_of_node(fwspec->fwnode)) + return -EINVAL; + + switch (fwspec->param[0]) { + case AIC_IRQ: + if (fwspec->param[1] >= ic->nr_hw) + return -EINVAL; + *hwirq = fwspec->param[1]; + break; + case AIC_FIQ: + if (fwspec->param[1] >= AIC_NR_FIQ) + return -EINVAL; + *hwirq = ic->nr_hw + fwspec->param[1]; + + /* + * In EL1 the non-redirected registers are the guest's, + * not EL2's, so remap the hwirqs to match. + */ + if (!is_kernel_in_hyp_mode()) { + switch (fwspec->param[1]) { + case AIC_TMR_GUEST_PHYS: + *hwirq = ic->nr_hw + AIC_TMR_EL0_PHYS; + break; + case AIC_TMR_GUEST_VIRT: + *hwirq = ic->nr_hw + AIC_TMR_EL0_VIRT; + break; + case AIC_TMR_HV_PHYS: + case AIC_TMR_HV_VIRT: + return -ENOENT; + default: + break; + } + } + break; + default: + return -EINVAL; + } + + *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; + + return 0; +} + +static int aic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + unsigned int type = IRQ_TYPE_NONE; + struct irq_fwspec *fwspec = arg; + irq_hw_number_t hwirq; + int i, ret; + + ret = aic_irq_domain_translate(domain, fwspec, &hwirq, &type); + if (ret) + return ret; + + for (i = 0; i < nr_irqs; i++) { + ret = aic_irq_domain_map(domain, virq + i, hwirq + i); + if (ret) + return ret; + } + + return 0; +} + +static void aic_irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } +} + +static const struct irq_domain_ops aic_irq_domain_ops = { + .translate = aic_irq_domain_translate, + .alloc = aic_irq_domain_alloc, + .free = aic_irq_domain_free, +}; + +/* + * IPI irqchip + */ + +static void aic_ipi_mask(struct irq_data *d) +{ + u32 irq_bit = BIT(irqd_to_hwirq(d)); + + /* No specific ordering requirements needed here. */ + atomic_andnot(irq_bit, this_cpu_ptr(&aic_vipi_enable)); +} + +static void aic_ipi_unmask(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + u32 irq_bit = BIT(irqd_to_hwirq(d)); + + atomic_or(irq_bit, this_cpu_ptr(&aic_vipi_enable)); + + /* + * The atomic_or() above must complete before the atomic_read() + * below to avoid racing aic_ipi_send_mask(). + */ + smp_mb__after_atomic(); + + /* + * If a pending vIPI was unmasked, raise a HW IPI to ourselves. + * No barriers needed here since this is a self-IPI. + */ + if (atomic_read(this_cpu_ptr(&aic_vipi_flag)) & irq_bit) + aic_ic_write(ic, AIC_IPI_SEND, AIC_IPI_SEND_CPU(smp_processor_id())); +} + +static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + u32 irq_bit = BIT(irqd_to_hwirq(d)); + u32 send = 0; + int cpu; + unsigned long pending; + + for_each_cpu(cpu, mask) { + /* + * This sequence is the mirror of the one in aic_ipi_unmask(); + * see the comment there. Additionally, release semantics + * ensure that the vIPI flag set is ordered after any shared + * memory accesses that precede it. This therefore also pairs + * with the atomic_fetch_andnot in aic_handle_ipi(). + */ + pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&aic_vipi_flag, cpu)); + + /* + * The atomic_fetch_or_release() above must complete before the + * atomic_read() below to avoid racing aic_ipi_unmask(). + */ + smp_mb__after_atomic(); + + if (!(pending & irq_bit) && + (atomic_read(per_cpu_ptr(&aic_vipi_enable, cpu)) & irq_bit)) + send |= AIC_IPI_SEND_CPU(cpu); + } + + /* + * The flag writes must complete before the physical IPI is issued + * to another CPU. This is implied by the control dependency on + * the result of atomic_read_acquire() above, which is itself + * already ordered after the vIPI flag write. + */ + if (send) + aic_ic_write(ic, AIC_IPI_SEND, send); +} + +static struct irq_chip ipi_chip = { + .name = "AIC-IPI", + .irq_mask = aic_ipi_mask, + .irq_unmask = aic_ipi_unmask, + .ipi_send_mask = aic_ipi_send_mask, +}; + +/* + * IPI IRQ domain + */ + +static void aic_handle_ipi(struct pt_regs *regs) +{ + int i; + unsigned long enabled, firing; + + /* + * Ack the IPI. We need to order this after the AIC event read, but + * that is enforced by normal MMIO ordering guarantees. + */ + aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_OTHER); + + /* + * The mask read does not need to be ordered. Only we can change + * our own mask anyway, so no races are possible here, as long as + * we are properly in the interrupt handler (which is covered by + * the barrier that is part of the top-level AIC handler's readl()). + */ + enabled = atomic_read(this_cpu_ptr(&aic_vipi_enable)); + + /* + * Clear the IPIs we are about to handle. This pairs with the + * atomic_fetch_or_release() in aic_ipi_send_mask(), and needs to be + * ordered after the aic_ic_write() above (to avoid dropping vIPIs) and + * before IPI handling code (to avoid races handling vIPIs before they + * are signaled). The former is taken care of by the release semantics + * of the write portion, while the latter is taken care of by the + * acquire semantics of the read portion. + */ + firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&aic_vipi_flag)) & enabled; + + for_each_set_bit(i, &firing, AIC_NR_SWIPI) + handle_domain_irq(aic_irqc->ipi_domain, i, regs); + + /* + * No ordering needed here; at worst this just changes the timing of + * when the next IPI will be delivered. + */ + aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); +} + +static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + irq_set_percpu_devid(virq + i); + irq_domain_set_info(d, virq + i, i, &ipi_chip, d->host_data, + handle_percpu_devid_irq, NULL, NULL); + } + + return 0; +} + +static void aic_ipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) +{ + /* Not freeing IPIs */ +} + +static const struct irq_domain_ops aic_ipi_domain_ops = { + .alloc = aic_ipi_alloc, + .free = aic_ipi_free, +}; + +static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) +{ + struct irq_domain *ipi_domain; + int base_ipi; + + ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_SWIPI, + &aic_ipi_domain_ops, irqc); + if (WARN_ON(!ipi_domain)) + return -ENODEV; + + ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; + irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); + + base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI, + NUMA_NO_NODE, NULL, false, NULL); + + if (WARN_ON(base_ipi < 0)) { + irq_domain_remove(ipi_domain); + return -ENODEV; +static void irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } +} + +static const struct irq_domain_ops irq_domain_ops = { + .translate = irq_domain_translate, + .alloc = irq_domain_alloc, + .free = irq_domain_free, +}; + +static const struct irq_domain_ops ipi_domain_ops = { + .alloc = ipi_domain_alloc, + .free = irq_domain_free, +}; + +static int aic_init_cpu(unsigned int cpu) +{ + /* Mask all hard-wired per-CPU IRQ/FIQ sources */ + + /* Pending Fast IPI FIQs */ + write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); + + /* Timer FIQs */ + sysreg_clear_set(cntp_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); + sysreg_clear_set(cntv_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); + + /* EL2-only (VHE mode) IRQ sources */ + if (is_kernel_in_hyp_mode()) { + /* Guest timers */ + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, + VM_TMR_FIQ_ENABLE_V | VM_TMR_FIQ_ENABLE_P, 0); + + /* vGIC maintenance IRQ */ + sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); + } + + /* PMC FIQ */ + sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, + FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); + + /* Uncore PMC FIQ */ + sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, + FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); + + /* Commit all of the above */ + isb(); + + /* + * Make sure the kernel's idea of logical CPU order is the same as AIC's + * If we ever end up with a mismatch here, we will have to introduce + * a mapping table similar to what other irqchip drivers do. + */ + WARN_ON(aic_ic_read(aic_irqc, AIC_WHOAMI) != smp_processor_id()); + + /* + * Always keep IPIs unmasked at the hardware level (except auto-masking + * by AIC during processing). We manage masks at the vIPI level. + */ + aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_SELF | AIC_IPI_OTHER); + aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_SELF); + aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); + + /* Initialize the local mask state */ + __this_cpu_write(aic_fiq_unmasked, 0); + + return 0; +} + +static struct gic_kvm_info vgic_info __initdata = { + .type = GIC_V3, + .no_maint_irq_mask = true, + .no_hw_deactivation = true, +}; + +static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) +{ + int i; + void __iomem *regs; + u32 info; + struct aic_irq_chip *irqc; + + regs = of_iomap(node, 0); + if (WARN_ON(!regs)) + return -EIO; + + irqc = kzalloc(sizeof(*irqc), GFP_KERNEL); + if (!irqc) + return -ENOMEM; + + aic_irqc = irqc; + irqc->base = regs; + + info = aic_ic_read(irqc, AIC_INFO); + irqc->nr_hw = FIELD_GET(AIC_INFO_NR_HW, info); + + irqc->hw_domain = irq_domain_create_linear(of_node_to_fwnode(node), + irqc->nr_hw + AIC_NR_FIQ, + &aic_irq_domain_ops, irqc); + if (WARN_ON(!irqc->hw_domain)) { + iounmap(irqc->base); + kfree(irqc); + return -ENODEV; + } + + irq_domain_update_bus_token(irqc->hw_domain, DOMAIN_BUS_WIRED); + + if (aic_init_smp(irqc, node)) { + irq_domain_remove(irqc->hw_domain); + iounmap(irqc->base); + kfree(irqc); + return -ENODEV; + } + + set_handle_irq(aic_handle_irq); + set_handle_fiq(aic_handle_fiq); + + for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) + aic_ic_write(irqc, AIC_MASK_SET + i * 4, U32_MAX); + for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) + aic_ic_write(irqc, AIC_SW_CLR + i * 4, U32_MAX); + for (i = 0; i < irqc->nr_hw; i++) + aic_ic_write(irqc, AIC_TARGET_CPU + i * 4, 1); + + if (!is_kernel_in_hyp_mode()) + pr_info("Kernel running in EL1, mapping interrupts"); + + cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_AIC_STARTING, + "irqchip/apple-aic/ipi:starting", + aic_init_cpu, NULL); + + vgic_set_kvm_info(&vgic_info); + + pr_info("Initialized with %d IRQs, %d FIQs, %d vIPIs\n", + irqc->nr_hw, AIC_NR_FIQ, AIC_NR_SWIPI); + + return 0; +} + +IRQCHIP_DECLARE(apple_m1_aic, "apple,aic", aic_of_ic_init); From 1071811bed0741c98335178bb05ca4b87ffc78a8 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 19:42:49 +0000 Subject: [PATCH 07/16] arm64: vipi: delete unused code No further changes. Signed-off-by: Pip Cet --- arch/arm64/kernel/vipi.c | 657 --------------------------------------- 1 file changed, 657 deletions(-) diff --git a/arch/arm64/kernel/vipi.c b/arch/arm64/kernel/vipi.c index 41bf9d79031b59..b9f98abfc4a02b 100644 --- a/arch/arm64/kernel/vipi.c +++ b/arch/arm64/kernel/vipi.c @@ -8,49 +8,11 @@ * Copyright 2015 Broadcom */ -/* - * AIC is a fairly simple interrupt controller with the following features: - * - * - 896 level-triggered hardware IRQs - * - Single mask bit per IRQ - * - Per-IRQ affinity setting - * - Automatic masking on event delivery (auto-ack) - * - Software triggering (ORed with hw line) - * - 2 per-CPU IPIs (meant as "self" and "other", but they are - * interchangeable if not symmetric) - * - Automatic prioritization (single event/ack register per CPU, lower IRQs = - * higher priority) - * - Automatic masking on ack - * - Default "this CPU" register view and explicit per-CPU views - * - * In addition, this driver also handles FIQs, as these are routed to the same - * IRQ vector. These are used for Fast IPIs (TODO), the ARMv8 timer IRQs, and - * performance counters (TODO). - * - * Implementation notes: - * - * - This driver creates two IRQ domains, one for HW IRQs and internal FIQs, - * and one for IPIs. - * - Since Linux needs more than 2 IPIs, we implement a software IRQ controller - * and funnel all IPIs into one per-CPU IPI (the second "self" IPI is unused). - * - FIQ hwirq numbers are assigned after true hwirqs, and are per-cpu. - * - DT bindings use 3-cell form (like GIC): - * - <0 nr flags> - hwirq #nr - * - <1 nr flags> - FIQ #nr - * - nr=0 Physical HV timer - * - nr=1 Virtual HV timer - * - nr=2 Physical guest timer - * - nr=3 Virtual guest timer - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include #include #include -#include #include #include #include @@ -59,506 +21,6 @@ #include #include -#include - -/* - * AIC registers (MMIO) - */ - -#define AIC_INFO 0x0004 -#define AIC_INFO_NR_HW GENMASK(15, 0) - -#define AIC_CONFIG 0x0010 - -#define AIC_WHOAMI 0x2000 -#define AIC_EVENT 0x2004 -#define AIC_EVENT_TYPE GENMASK(31, 16) -#define AIC_EVENT_NUM GENMASK(15, 0) - -#define AIC_EVENT_TYPE_HW 1 -#define AIC_EVENT_TYPE_IPI 4 -#define AIC_EVENT_IPI_OTHER 1 -#define AIC_EVENT_IPI_SELF 2 - -#define AIC_IPI_SEND 0x2008 -#define AIC_IPI_ACK 0x200c -#define AIC_IPI_MASK_SET 0x2024 -#define AIC_IPI_MASK_CLR 0x2028 - -#define AIC_IPI_SEND_CPU(cpu) BIT(cpu) - -#define AIC_IPI_OTHER BIT(0) -#define AIC_IPI_SELF BIT(31) - -#define AIC_TARGET_CPU 0x3000 -#define AIC_SW_SET 0x4000 -#define AIC_SW_CLR 0x4080 -#define AIC_MASK_SET 0x4100 -#define AIC_MASK_CLR 0x4180 - -#define AIC_CPU_IPI_SET(cpu) (0x5008 + ((cpu) << 7)) -#define AIC_CPU_IPI_CLR(cpu) (0x500c + ((cpu) << 7)) -#define AIC_CPU_IPI_MASK_SET(cpu) (0x5024 + ((cpu) << 7)) -#define AIC_CPU_IPI_MASK_CLR(cpu) (0x5028 + ((cpu) << 7)) - -#define MASK_REG(x) (4 * ((x) >> 5)) -#define MASK_BIT(x) BIT((x) & GENMASK(4, 0)) - -/* - * IMP-DEF sysregs that control FIQ sources - * Note: sysreg-based IPIs are not supported yet. - */ - -/* Core PMC control register */ -#define SYS_IMP_APL_PMCR0_EL1 sys_reg(3, 1, 15, 0, 0) -#define PMCR0_IMODE GENMASK(10, 8) -#define PMCR0_IMODE_OFF 0 -#define PMCR0_IMODE_PMI 1 -#define PMCR0_IMODE_AIC 2 -#define PMCR0_IMODE_HALT 3 -#define PMCR0_IMODE_FIQ 4 -#define PMCR0_IACT BIT(11) - -/* IPI request registers */ -#define SYS_IMP_APL_IPI_RR_LOCAL_EL1 sys_reg(3, 5, 15, 0, 0) -#define SYS_IMP_APL_IPI_RR_GLOBAL_EL1 sys_reg(3, 5, 15, 0, 1) -#define IPI_RR_CPU GENMASK(7, 0) -/* Cluster only used for the GLOBAL register */ -#define IPI_RR_CLUSTER GENMASK(23, 16) -#define IPI_RR_TYPE GENMASK(29, 28) -#define IPI_RR_IMMEDIATE 0 -#define IPI_RR_RETRACT 1 -#define IPI_RR_DEFERRED 2 -#define IPI_RR_NOWAKE 3 - -/* IPI status register */ -#define SYS_IMP_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) -#define IPI_SR_PENDING BIT(0) - -/* Guest timer FIQ enable register */ -#define SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2 sys_reg(3, 5, 15, 1, 3) -#define VM_TMR_FIQ_ENABLE_V BIT(0) -#define VM_TMR_FIQ_ENABLE_P BIT(1) - -/* Deferred IPI countdown register */ -#define SYS_IMP_APL_IPI_CR_EL1 sys_reg(3, 5, 15, 3, 1) - -/* Uncore PMC control register */ -#define SYS_IMP_APL_UPMCR0_EL1 sys_reg(3, 7, 15, 0, 4) -#define UPMCR0_IMODE GENMASK(18, 16) -#define UPMCR0_IMODE_OFF 0 -#define UPMCR0_IMODE_AIC 2 -#define UPMCR0_IMODE_HALT 3 -#define UPMCR0_IMODE_FIQ 4 - -/* Uncore PMC status register */ -#define SYS_IMP_APL_UPMSR_EL1 sys_reg(3, 7, 15, 6, 4) -#define UPMSR_IACT BIT(0) - -#define AIC_NR_FIQ 4 -#define AIC_NR_SWIPI 32 - -/* - * FIQ hwirq index definitions: FIQ sources use the DT binding defines - * directly, except that timers are special. At the irqchip level, the - * two timer types are represented by their access method: _EL0 registers - * or _EL02 registers. In the DT binding, the timers are represented - * by their purpose (HV or guest). This mapping is for when the kernel is - * running at EL2 (with VHE). When the kernel is running at EL1, the - * mapping differs and aic_irq_domain_translate() performs the remapping. - */ - -#define AIC_TMR_EL0_PHYS AIC_TMR_HV_PHYS -#define AIC_TMR_EL0_VIRT AIC_TMR_HV_VIRT -#define AIC_TMR_EL02_PHYS AIC_TMR_GUEST_PHYS -#define AIC_TMR_EL02_VIRT AIC_TMR_GUEST_VIRT - -struct aic_irq_chip { - void __iomem *base; - struct irq_domain *hw_domain; - struct irq_domain *ipi_domain; - int nr_hw; - int ipi_hwirq; -}; - -static DEFINE_PER_CPU(uint32_t, aic_fiq_unmasked); - -static DEFINE_PER_CPU(atomic_t, aic_vipi_flag); -static DEFINE_PER_CPU(atomic_t, aic_vipi_enable); - -static struct aic_irq_chip *aic_irqc; - -static void aic_handle_ipi(struct pt_regs *regs); - -static u32 aic_ic_read(struct aic_irq_chip *ic, u32 reg) -{ - return readl_relaxed(ic->base + reg); -} - -static void aic_ic_write(struct aic_irq_chip *ic, u32 reg, u32 val) -{ - writel_relaxed(val, ic->base + reg); -} - -/* - * IRQ irqchip - */ - -static void aic_irq_mask(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - - aic_ic_write(ic, AIC_MASK_SET + MASK_REG(irqd_to_hwirq(d)), - MASK_BIT(irqd_to_hwirq(d))); -} - -static void aic_irq_unmask(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - - aic_ic_write(ic, AIC_MASK_CLR + MASK_REG(irqd_to_hwirq(d)), - MASK_BIT(irqd_to_hwirq(d))); -} - -static void aic_irq_eoi(struct irq_data *d) -{ - /* - * Reading the interrupt reason automatically acknowledges and masks - * the IRQ, so we just unmask it here if needed. - */ - if (!irqd_irq_masked(d)) - aic_irq_unmask(d); -} - -static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) -{ - struct aic_irq_chip *ic = aic_irqc; - u32 event, type, irq; - - do { - /* - * We cannot use a relaxed read here, as reads from DMA buffers - * need to be ordered after the IRQ fires. - */ - event = readl(ic->base + AIC_EVENT); - type = FIELD_GET(AIC_EVENT_TYPE, event); - irq = FIELD_GET(AIC_EVENT_NUM, event); - - if (type == AIC_EVENT_TYPE_HW) - handle_domain_irq(ic->hw_domain, irq, regs); - else if (type == AIC_EVENT_TYPE_IPI) - aic_handle_ipi(0 /* irq */, regs); - else if (event != 0) - pr_err_ratelimited("Unknown IRQ event %d, %d\n", type, irq); - } while (event); - - /* - * vGIC maintenance interrupts end up here too, so we need to check - * for them separately. This should never trigger if KVM is working - * properly, because it will have already taken care of clearing it - * on guest exit before this handler runs. - */ - if (is_kernel_in_hyp_mode() && (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EN) && - read_sysreg_s(SYS_ICH_MISR_EL2) != 0) { - pr_err_ratelimited("vGIC IRQ fired and not handled by KVM, disabling.\n"); - sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); - } -} - -static int aic_irq_set_affinity(struct irq_data *d, - const struct cpumask *mask_val, bool force) -{ - irq_hw_number_t hwirq = irqd_to_hwirq(d); - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - int cpu; - u32 mask = 0; - - for_each_cpu(cpu, mask_val) - mask |= BIT(cpu); - - aic_ic_write(ic, AIC_TARGET_CPU + hwirq * 4, mask); - irq_data_update_effective_affinity(d, mask_val); - - return IRQ_SET_MASK_OK; -} - -static int aic_irq_set_type(struct irq_data *d, unsigned int type) -{ - /* - * Some IRQs (e.g. MSIs) implicitly have edge semantics, and we don't - * have a way to find out the type of any given IRQ, so just allow both. - */ - return (type == IRQ_TYPE_LEVEL_HIGH || type == IRQ_TYPE_EDGE_RISING) ? 0 : -EINVAL; -} - -static struct irq_chip aic_chip = { - .name = "AIC", - .irq_mask = aic_irq_mask, - .irq_unmask = aic_irq_unmask, - .irq_eoi = aic_irq_eoi, - .irq_set_affinity = aic_irq_set_affinity, - .irq_set_type = aic_irq_set_type, -}; - -/* - * FIQ irqchip - */ - -static unsigned long aic_fiq_get_idx(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - - return irqd_to_hwirq(d) - ic->nr_hw; -} - -static void aic_fiq_set_mask(struct irq_data *d) -{ - /* Only the guest timers have real mask bits, unfortunately. */ - switch (aic_fiq_get_idx(d)) { - case AIC_TMR_EL02_PHYS: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_P, 0); - isb(); - break; - case AIC_TMR_EL02_VIRT: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_V, 0); - isb(); - break; - default: - break; - } -} - -static void aic_fiq_clear_mask(struct irq_data *d) -{ - switch (aic_fiq_get_idx(d)) { - case AIC_TMR_EL02_PHYS: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_P); - isb(); - break; - case AIC_TMR_EL02_VIRT: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_V); - isb(); - break; - default: - break; - } -} - -static void aic_fiq_mask(struct irq_data *d) -{ - aic_fiq_set_mask(d); - __this_cpu_and(aic_fiq_unmasked, ~BIT(aic_fiq_get_idx(d))); -} - -static void aic_fiq_unmask(struct irq_data *d) -{ - aic_fiq_clear_mask(d); - __this_cpu_or(aic_fiq_unmasked, BIT(aic_fiq_get_idx(d))); -} - -static void aic_fiq_eoi(struct irq_data *d) -{ - /* We mask to ack (where we can), so we need to unmask at EOI. */ - if (__this_cpu_read(aic_fiq_unmasked) & BIT(aic_fiq_get_idx(d))) - aic_fiq_clear_mask(d); -} - -#define TIMER_FIRING(x) \ - (((x) & (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_MASK | \ - ARCH_TIMER_CTRL_IT_STAT)) == \ - (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_STAT)) - -static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) -{ - /* - * It would be really nice if we had a system register that lets us get - * the FIQ source state without having to peek down into sources... - * but such a register does not seem to exist. - * - * So, we have these potential sources to test for: - * - Fast IPIs (not yet used) - * - The 4 timers (CNTP, CNTV for each of HV and guest) - * - Per-core PMCs (not yet supported) - * - Per-cluster uncore PMCs (not yet supported) - * - * Since not dealing with any of these results in a FIQ storm, - * we check for everything here, even things we don't support yet. - */ - - if (read_sysreg_s(SYS_IMP_APL_IPI_SR_EL1) & IPI_SR_PENDING) { - pr_err_ratelimited("Fast IPI fired. Acking.\n"); - write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); - } - - if (TIMER_FIRING(read_sysreg(cntp_ctl_el0))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL0_PHYS, regs); - - if (TIMER_FIRING(read_sysreg(cntv_ctl_el0))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL0_VIRT, regs); - - if (is_kernel_in_hyp_mode()) { - uint64_t enabled = read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2); - - if ((enabled & VM_TMR_FIQ_ENABLE_P) && - TIMER_FIRING(read_sysreg_s(SYS_CNTP_CTL_EL02))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL02_PHYS, regs); - - if ((enabled & VM_TMR_FIQ_ENABLE_V) && - TIMER_FIRING(read_sysreg_s(SYS_CNTV_CTL_EL02))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL02_VIRT, regs); - } - - if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == - (FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_FIQ) | PMCR0_IACT)) { - /* - * Not supported yet, let's figure out how to handle this when - * we implement these proprietary performance counters. For now, - * just mask it and move on. - */ - pr_err_ratelimited("PMC FIQ fired. Masking.\n"); - sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, - FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); - } - - if (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ && - (read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) { - /* Same story with uncore PMCs */ - pr_err_ratelimited("Uncore PMC FIQ fired. Masking.\n"); - sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, - FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); - } -} - -static int aic_fiq_set_type(struct irq_data *d, unsigned int type) -{ - return (type == IRQ_TYPE_LEVEL_HIGH) ? 0 : -EINVAL; -} - -static struct irq_chip fiq_chip = { - .name = "AIC-FIQ", - .irq_mask = aic_fiq_mask, - .irq_unmask = aic_fiq_unmask, - .irq_ack = aic_fiq_set_mask, - .irq_eoi = aic_fiq_eoi, - .irq_set_type = aic_fiq_set_type, -}; - -/* - * Main IRQ domain - */ - -static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, - irq_hw_number_t hw) -{ - struct aic_irq_chip *ic = id->host_data; - - if (hw < ic->nr_hw) { - irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, - handle_fasteoi_irq, NULL, NULL); - irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); - } else { - irq_set_percpu_devid(irq); - irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, - handle_percpu_devid_irq, NULL, NULL); - } - - return 0; -} - -static int aic_irq_domain_translate(struct irq_domain *id, - struct irq_fwspec *fwspec, - unsigned long *hwirq, - unsigned int *type) -{ - struct aic_irq_chip *ic = id->host_data; - - if (fwspec->param_count != 3 || !is_of_node(fwspec->fwnode)) - return -EINVAL; - - switch (fwspec->param[0]) { - case AIC_IRQ: - if (fwspec->param[1] >= ic->nr_hw) - return -EINVAL; - *hwirq = fwspec->param[1]; - break; - case AIC_FIQ: - if (fwspec->param[1] >= AIC_NR_FIQ) - return -EINVAL; - *hwirq = ic->nr_hw + fwspec->param[1]; - - /* - * In EL1 the non-redirected registers are the guest's, - * not EL2's, so remap the hwirqs to match. - */ - if (!is_kernel_in_hyp_mode()) { - switch (fwspec->param[1]) { - case AIC_TMR_GUEST_PHYS: - *hwirq = ic->nr_hw + AIC_TMR_EL0_PHYS; - break; - case AIC_TMR_GUEST_VIRT: - *hwirq = ic->nr_hw + AIC_TMR_EL0_VIRT; - break; - case AIC_TMR_HV_PHYS: - case AIC_TMR_HV_VIRT: - return -ENOENT; - default: - break; - } - } - break; - default: - return -EINVAL; - } - - *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; - - return 0; -} - -static int aic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs, void *arg) -{ - unsigned int type = IRQ_TYPE_NONE; - struct irq_fwspec *fwspec = arg; - irq_hw_number_t hwirq; - int i, ret; - - ret = aic_irq_domain_translate(domain, fwspec, &hwirq, &type); - if (ret) - return ret; - - for (i = 0; i < nr_irqs; i++) { - ret = aic_irq_domain_map(domain, virq + i, hwirq + i); - if (ret) - return ret; - } - - return 0; -} - -static void aic_irq_domain_free(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs) -{ - int i; - - for (i = 0; i < nr_irqs; i++) { - struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); - - irq_set_handler(virq + i, NULL); - irq_domain_reset_irq_data(d); - } -} - -static const struct irq_domain_ops aic_irq_domain_ops = { - .translate = aic_irq_domain_translate, - .alloc = aic_irq_domain_alloc, - .free = aic_irq_domain_free, -}; - /* * IPI irqchip */ @@ -647,12 +109,6 @@ static void aic_handle_ipi(struct pt_regs *regs) int i; unsigned long enabled, firing; - /* - * Ack the IPI. We need to order this after the AIC event read, but - * that is enforced by normal MMIO ordering guarantees. - */ - aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_OTHER); - /* * The mask read does not need to be ordered. Only we can change * our own mask anyway, so no races are possible here, as long as @@ -675,11 +131,6 @@ static void aic_handle_ipi(struct pt_regs *regs) for_each_set_bit(i, &firing, AIC_NR_SWIPI) handle_domain_irq(aic_irqc->ipi_domain, i, regs); - /* - * No ordering needed here; at worst this just changes the timing of - * when the next IPI will be delivered. - */ - aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); } static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, @@ -736,125 +187,17 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) static int aic_init_cpu(unsigned int cpu) { - /* Mask all hard-wired per-CPU IRQ/FIQ sources */ - - /* Pending Fast IPI FIQs */ - write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); - - /* Timer FIQs */ - sysreg_clear_set(cntp_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); - sysreg_clear_set(cntv_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); - - /* EL2-only (VHE mode) IRQ sources */ - if (is_kernel_in_hyp_mode()) { - /* Guest timers */ - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, - VM_TMR_FIQ_ENABLE_V | VM_TMR_FIQ_ENABLE_P, 0); - - /* vGIC maintenance IRQ */ - sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); - } - - /* PMC FIQ */ - sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, - FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); - - /* Uncore PMC FIQ */ - sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, - FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); - - /* Commit all of the above */ - isb(); - - /* - * Make sure the kernel's idea of logical CPU order is the same as AIC's - * If we ever end up with a mismatch here, we will have to introduce - * a mapping table similar to what other irqchip drivers do. - */ - WARN_ON(aic_ic_read(aic_irqc, AIC_WHOAMI) != smp_processor_id()); - - /* - * Always keep IPIs unmasked at the hardware level (except auto-masking - * by AIC during processing). We manage masks at the vIPI level. - */ - aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_SELF | AIC_IPI_OTHER); - aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_SELF); - aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); - - /* Initialize the local mask state */ - __this_cpu_write(aic_fiq_unmasked, 0); - - return 0; -} - -static struct gic_kvm_info vgic_info __initdata = { - .type = GIC_V3, - .no_maint_irq_mask = true, - .no_hw_deactivation = true, -}; - -static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) -{ - int i; - void __iomem *regs; - u32 info; - struct aic_irq_chip *irqc; - - regs = of_iomap(node, 0); - if (WARN_ON(!regs)) - return -EIO; irqc = kzalloc(sizeof(*irqc), GFP_KERNEL); if (!irqc) return -ENOMEM; - aic_irqc = irqc; - irqc->base = regs; - info = aic_ic_read(irqc, AIC_INFO); - irqc->nr_hw = FIELD_GET(AIC_INFO_NR_HW, info); - irqc->hw_domain = irq_domain_create_linear(of_node_to_fwnode(node), - irqc->nr_hw + AIC_NR_FIQ, - &aic_irq_domain_ops, irqc); - if (WARN_ON(!irqc->hw_domain)) { - iounmap(irqc->base); - kfree(irqc); - return -ENODEV; - } - irq_domain_update_bus_token(irqc->hw_domain, DOMAIN_BUS_WIRED); - - if (aic_init_smp(irqc, node)) { - irq_domain_remove(irqc->hw_domain); - iounmap(irqc->base); - kfree(irqc); - return -ENODEV; - } - - set_handle_irq(aic_handle_irq); - set_handle_fiq(aic_handle_fiq); - - for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) - aic_ic_write(irqc, AIC_MASK_SET + i * 4, U32_MAX); - for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) - aic_ic_write(irqc, AIC_SW_CLR + i * 4, U32_MAX); - for (i = 0; i < irqc->nr_hw; i++) - aic_ic_write(irqc, AIC_TARGET_CPU + i * 4, 1); - - if (!is_kernel_in_hyp_mode()) - pr_info("Kernel running in EL1, mapping interrupts"); - - cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_AIC_STARTING, - "irqchip/apple-aic/ipi:starting", - aic_init_cpu, NULL); - - vgic_set_kvm_info(&vgic_info); pr_info("Initialized with %d IRQs, %d FIQs, %d vIPIs\n", irqc->nr_hw, AIC_NR_FIQ, AIC_NR_SWIPI); return 0; } - -IRQCHIP_DECLARE(apple_m1_aic, "apple,aic", aic_of_ic_init); From 9dfd8644a69550f444df071fd14cb015a167b3cc Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 19:46:11 +0000 Subject: [PATCH 08/16] arm64: vipi: mechanical replacement of AIC prefixes by vipi ones No functional code changes intended. Signed-off-by: Pip Cet --- arch/arm64/kernel/vipi.c | 88 +++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/arch/arm64/kernel/vipi.c b/arch/arm64/kernel/vipi.c index b9f98abfc4a02b..bfb9f7dfb31c1b 100644 --- a/arch/arm64/kernel/vipi.c +++ b/arch/arm64/kernel/vipi.c @@ -25,20 +25,20 @@ * IPI irqchip */ -static void aic_ipi_mask(struct irq_data *d) +static void vipi_mask(struct irq_data *d) { u32 irq_bit = BIT(irqd_to_hwirq(d)); /* No specific ordering requirements needed here. */ - atomic_andnot(irq_bit, this_cpu_ptr(&aic_vipi_enable)); + atomic_andnot(irq_bit, this_cpu_ptr(&vipi_enable)); } -static void aic_ipi_unmask(struct irq_data *d) +static void vipi_unmask(struct irq_data *d) { - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + struct vipi_irq_chip *ic = irq_data_get_irq_chip_data(d); u32 irq_bit = BIT(irqd_to_hwirq(d)); - atomic_or(irq_bit, this_cpu_ptr(&aic_vipi_enable)); + atomic_or(irq_bit, this_cpu_ptr(&vipi_enable)); /* * The atomic_or() above must complete before the atomic_read() @@ -50,31 +50,35 @@ static void aic_ipi_unmask(struct irq_data *d) * If a pending vIPI was unmasked, raise a HW IPI to ourselves. * No barriers needed here since this is a self-IPI. */ - if (atomic_read(this_cpu_ptr(&aic_vipi_flag)) & irq_bit) - aic_ic_write(ic, AIC_IPI_SEND, AIC_IPI_SEND_CPU(smp_processor_id())); + if (atomic_read(this_cpu_ptr(&vipi_flag)) & irq_bit) { + struct cpumask self_mask = { 0, }; + cpumask_set_cpu(smp_processor_id(), &self_mask); + ipi_send_mask(ic->hwirq->irq, &self_mask); + } } -static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) +static void vipi_send_mask(struct irq_data *d, const struct cpumask *mask) { - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + struct vipi_irq_chip *ic = irq_data_get_irq_chip_data(d); u32 irq_bit = BIT(irqd_to_hwirq(d)); - u32 send = 0; int cpu; + bool send = false; unsigned long pending; + struct cpumask sendmask = *mask; for_each_cpu(cpu, mask) { /* - * This sequence is the mirror of the one in aic_ipi_unmask(); + * This sequence is the mirror of the one in vipi_unmask(); * see the comment there. Additionally, release semantics * ensure that the vIPI flag set is ordered after any shared * memory accesses that precede it. This therefore also pairs - * with the atomic_fetch_andnot in aic_handle_ipi(). + * with the atomic_fetch_andnot in handle_ipi(). */ - pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&aic_vipi_flag, cpu)); + pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&vipi_flag, cpu)); /* * The atomic_fetch_or_release() above must complete before the - * atomic_read() below to avoid racing aic_ipi_unmask(). + * atomic_read() below to avoid racing vipi_unmask(). */ smp_mb__after_atomic(); @@ -93,18 +97,18 @@ static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) aic_ic_write(ic, AIC_IPI_SEND, send); } -static struct irq_chip ipi_chip = { - .name = "AIC-IPI", - .irq_mask = aic_ipi_mask, - .irq_unmask = aic_ipi_unmask, - .ipi_send_mask = aic_ipi_send_mask, +static struct irq_chip vipi_chip = { + .name = "VIPI", + .irq_mask = vipi_mask, + .irq_unmask = vipi_unmask, + .ipi_send_mask = vipi_send_mask, }; /* * IPI IRQ domain */ -static void aic_handle_ipi(struct pt_regs *regs) +static void handle_ipi(struct irq_desc *d) { int i; unsigned long enabled, firing; @@ -115,72 +119,72 @@ static void aic_handle_ipi(struct pt_regs *regs) * we are properly in the interrupt handler (which is covered by * the barrier that is part of the top-level AIC handler's readl()). */ - enabled = atomic_read(this_cpu_ptr(&aic_vipi_enable)); + enabled = atomic_read(this_cpu_ptr(&vipi_enable)); /* * Clear the IPIs we are about to handle. This pairs with the - * atomic_fetch_or_release() in aic_ipi_send_mask(), and needs to be - * ordered after the aic_ic_write() above (to avoid dropping vIPIs) and + * atomic_fetch_or_release() in vipi_send_mask(), and needs to be + * ordered after the ic_write() above (to avoid dropping vIPIs) and * before IPI handling code (to avoid races handling vIPIs before they * are signaled). The former is taken care of by the release semantics * of the write portion, while the latter is taken care of by the * acquire semantics of the read portion. */ - firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&aic_vipi_flag)) & enabled; + firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&vipi_flag)) & enabled; for_each_set_bit(i, &firing, AIC_NR_SWIPI) handle_domain_irq(aic_irqc->ipi_domain, i, regs); } -static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, - unsigned int nr_irqs, void *args) +static int vipi_alloc(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs, void *args) { int i; for (i = 0; i < nr_irqs; i++) { irq_set_percpu_devid(virq + i); - irq_domain_set_info(d, virq + i, i, &ipi_chip, d->host_data, + irq_domain_set_info(d, virq + i, i, &vipi_chip, d->host_data, handle_percpu_devid_irq, NULL, NULL); } return 0; } -static void aic_ipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) +static void vipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) { /* Not freeing IPIs */ } -static const struct irq_domain_ops aic_ipi_domain_ops = { - .alloc = aic_ipi_alloc, - .free = aic_ipi_free, +static const struct irq_domain_ops vipi_domain_ops = { + .alloc = vipi_alloc, + .free = vipi_free, }; -static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) +static int vipi_init_smp(struct vipi_irq_chip *irqc) { - struct irq_domain *ipi_domain; + struct irq_domain *vipi_domain; int base_ipi; ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_SWIPI, - &aic_ipi_domain_ops, irqc); - if (WARN_ON(!ipi_domain)) - return -ENODEV; + &vipi_domain_ops, irqc); + if (WARN_ON(!vipi_domain)) + return -ENOMEM; - ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; - irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); + vipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; + irq_domain_update_bus_token(vipi_domain, DOMAIN_BUS_IPI); - base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI, + base_ipi = __irq_domain_alloc_irqs(vipi_domain, -1, NR_SWIPI, NUMA_NO_NODE, NULL, false, NULL); if (WARN_ON(base_ipi < 0)) { - irq_domain_remove(ipi_domain); + irq_domain_remove(vipi_domain); return -ENODEV; } - set_smp_ipi_range(base_ipi, AIC_NR_SWIPI); + set_smp_ipi_range(base_ipi, NR_SWIPI); - irqc->ipi_domain = ipi_domain; + irqc->domain = vipi_domain; return 0; } From 28a9ae246e6f38ab4b3264e7cf8a11d9f47535ad Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 18:24:09 +0000 Subject: [PATCH 09/16] arm64: vipi: vIPI code adjustments Some systems, such as the Apple M1 SoC, do not have more than one or two hardware IPIs that can be distinguished and masked independently. For such systems, it is necessary to provide "virtual" IPIs, but since there are at least two IRQ controllers that need this feature, let's move it into common code. The new entrypoint accepts a potentially non-maskable single per-CPU IPI interrupt and provides up to 32 "virtual" per-CPU IPIs that are maskable, can be distinguished, and live in an IPI domain. set_smp_ipi_range is changed to transparently enable the vIPI layer on IPI-challenged systems. TODO: move vipi_init definition to header file TODO: return a value from set_smp_ipi_range so drivers know whether to set irq domain flags. TODO: make conditional on IPI-challenged archs. Signed-off-by: Pip Cet --- arch/arm64/Kconfig | 3 +++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/smp.c | 16 +++++++++++++ arch/arm64/kernel/vipi.c | 48 ++++++++++++++++++++++++++++++-------- 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b5b13a932561fc..9a7b03cbd6ae1e 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -314,6 +314,9 @@ config ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE config SMP def_bool y +config IPI_FUNNELING + def_bool y + config KERNEL_MODE_NEON def_bool y diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 3f1490bfb938a0..ed58d15d484c2e 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -73,6 +73,7 @@ obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o obj-$(CONFIG_ARM64_MTE) += mte.o obj-y += vdso-wrap.o obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o +obj-$(CONFIG_IPI_FUNNELING) += vipi.o obj-y += probes/ head-y := head.o diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 6f6ff072acbde7..ac5f26281d347c 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -979,10 +979,26 @@ static void ipi_teardown(int cpu) } #endif +#ifdef CONFIG_IPI_FUNNELING +extern int __init vipi_init(struct irq_data *hwirq); +#else +static inline int vipi_init(irqdata) +{ + return -EINVAL; +} +#endif + void __init set_smp_ipi_range(int ipi_base, int n) { int i; + if (n < NR_IPI) { + int ret; + BUG_ON(n < 1); + ret = vipi_init(irq_get_irq_data(ipi_base)); + if (ret >= 0) + return; + } WARN_ON(n < NR_IPI); nr_ipi = min(n, NR_IPI); diff --git a/arch/arm64/kernel/vipi.c b/arch/arm64/kernel/vipi.c index bfb9f7dfb31c1b..863849cf2e0c51 100644 --- a/arch/arm64/kernel/vipi.c +++ b/arch/arm64/kernel/vipi.c @@ -21,6 +21,19 @@ #include #include +struct vipi_irq_chip { + struct irq_domain *domain; + struct irq_data *hwirq; +}; + +#define NR_SWIPI 32 + +static DEFINE_PER_CPU(atomic_t, vipi_flag); +static DEFINE_PER_CPU(atomic_t, vipi_enable); + +static struct vipi_irq_chip *vipi_irqc; + +static void handle_ipi(struct irq_desc *d); /* * IPI irqchip */ @@ -83,8 +96,10 @@ static void vipi_send_mask(struct irq_data *d, const struct cpumask *mask) smp_mb__after_atomic(); if (!(pending & irq_bit) && - (atomic_read(per_cpu_ptr(&aic_vipi_enable, cpu)) & irq_bit)) - send |= AIC_IPI_SEND_CPU(cpu); + (atomic_read(per_cpu_ptr(&vipi_enable, cpu)) & irq_bit)) { + cpumask_set_cpu(cpu, &sendmask); + send = true; + } } /* @@ -94,7 +109,7 @@ static void vipi_send_mask(struct irq_data *d, const struct cpumask *mask) * already ordered after the vIPI flag write. */ if (send) - aic_ic_write(ic, AIC_IPI_SEND, send); + ipi_send_mask(ic->hwirq->irq, &sendmask); } static struct irq_chip vipi_chip = { @@ -132,9 +147,12 @@ static void handle_ipi(struct irq_desc *d) */ firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&vipi_flag)) & enabled; - for_each_set_bit(i, &firing, AIC_NR_SWIPI) - handle_domain_irq(aic_irqc->ipi_domain, i, regs); + for_each_set_bit(i, &firing, NR_SWIPI) { + struct irq_desc *nd = + irq_resolve_mapping(vipi_irqc->domain, i); + handle_irq_desc(nd); + } } static int vipi_alloc(struct irq_domain *d, unsigned int virq, @@ -154,6 +172,7 @@ static int vipi_alloc(struct irq_domain *d, unsigned int virq, static void vipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) { /* Not freeing IPIs */ + WARN_ON(1); } static const struct irq_domain_ops vipi_domain_ops = { @@ -165,8 +184,12 @@ static int vipi_init_smp(struct vipi_irq_chip *irqc) { struct irq_domain *vipi_domain; int base_ipi; + struct fwnode_handle *fwnode; + + fwnode = __irq_domain_alloc_fwnode(IRQCHIP_FWNODE_NAMED, 0, + "vIPI", NULL); - ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_SWIPI, + vipi_domain = irq_domain_create_linear(fwnode, NR_SWIPI, &vipi_domain_ops, irqc); if (WARN_ON(!vipi_domain)) return -ENOMEM; @@ -179,7 +202,7 @@ static int vipi_init_smp(struct vipi_irq_chip *irqc) if (WARN_ON(base_ipi < 0)) { irq_domain_remove(vipi_domain); - return -ENODEV; + return -ENOMEM; } set_smp_ipi_range(base_ipi, NR_SWIPI); @@ -189,19 +212,24 @@ static int vipi_init_smp(struct vipi_irq_chip *irqc) return 0; } -static int aic_init_cpu(unsigned int cpu) +int __init vipi_init(struct irq_data *hwirq) { + struct vipi_irq_chip *irqc; irqc = kzalloc(sizeof(*irqc), GFP_KERNEL); if (!irqc) return -ENOMEM; + irqc->hwirq = hwirq; + if (vipi_init_smp(irqc)) + return -ENOMEM; + vipi_irqc = irqc; + irq_set_handler_locked(hwirq, handle_ipi); - pr_info("Initialized with %d IRQs, %d FIQs, %d vIPIs\n", - irqc->nr_hw, AIC_NR_FIQ, AIC_NR_SWIPI); + pr_info("Initialized with %d vIPIs\n", NR_SWIPI); return 0; } From dfbbe1e64db4deb1b40ea7121ecaddf64d2baf27 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 18:02:58 +0000 Subject: [PATCH 10/16] irq-apple-aic: delete unused code No further changes Signed-off-by: Pip Cet --- drivers/irqchip/irq-apple-aic.c | 282 ------------------------ drivers/irqchip/irq-apple-fiq.c | 378 -------------------------------- 2 files changed, 660 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 41bf9d79031b59..ce7bbd3ade23b0 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -23,10 +23,6 @@ * - Automatic masking on ack * - Default "this CPU" register view and explicit per-CPU views * - * In addition, this driver also handles FIQs, as these are routed to the same - * IRQ vector. These are used for Fast IPIs (TODO), the ARMv8 timer IRQs, and - * performance counters (TODO). - * * Implementation notes: * * - This driver creates two IRQ domains, one for HW IRQs and internal FIQs, @@ -36,11 +32,6 @@ * - FIQ hwirq numbers are assigned after true hwirqs, and are per-cpu. * - DT bindings use 3-cell form (like GIC): * - <0 nr flags> - hwirq #nr - * - <1 nr flags> - FIQ #nr - * - nr=0 Physical HV timer - * - nr=1 Virtual HV timer - * - nr=2 Physical guest timer - * - nr=3 Virtual guest timer */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -104,87 +95,13 @@ #define MASK_REG(x) (4 * ((x) >> 5)) #define MASK_BIT(x) BIT((x) & GENMASK(4, 0)) -/* - * IMP-DEF sysregs that control FIQ sources - * Note: sysreg-based IPIs are not supported yet. - */ - -/* Core PMC control register */ -#define SYS_IMP_APL_PMCR0_EL1 sys_reg(3, 1, 15, 0, 0) -#define PMCR0_IMODE GENMASK(10, 8) -#define PMCR0_IMODE_OFF 0 -#define PMCR0_IMODE_PMI 1 -#define PMCR0_IMODE_AIC 2 -#define PMCR0_IMODE_HALT 3 -#define PMCR0_IMODE_FIQ 4 -#define PMCR0_IACT BIT(11) - -/* IPI request registers */ -#define SYS_IMP_APL_IPI_RR_LOCAL_EL1 sys_reg(3, 5, 15, 0, 0) -#define SYS_IMP_APL_IPI_RR_GLOBAL_EL1 sys_reg(3, 5, 15, 0, 1) -#define IPI_RR_CPU GENMASK(7, 0) -/* Cluster only used for the GLOBAL register */ -#define IPI_RR_CLUSTER GENMASK(23, 16) -#define IPI_RR_TYPE GENMASK(29, 28) -#define IPI_RR_IMMEDIATE 0 -#define IPI_RR_RETRACT 1 -#define IPI_RR_DEFERRED 2 -#define IPI_RR_NOWAKE 3 - -/* IPI status register */ -#define SYS_IMP_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) -#define IPI_SR_PENDING BIT(0) - -/* Guest timer FIQ enable register */ -#define SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2 sys_reg(3, 5, 15, 1, 3) -#define VM_TMR_FIQ_ENABLE_V BIT(0) -#define VM_TMR_FIQ_ENABLE_P BIT(1) - -/* Deferred IPI countdown register */ -#define SYS_IMP_APL_IPI_CR_EL1 sys_reg(3, 5, 15, 3, 1) - -/* Uncore PMC control register */ -#define SYS_IMP_APL_UPMCR0_EL1 sys_reg(3, 7, 15, 0, 4) -#define UPMCR0_IMODE GENMASK(18, 16) -#define UPMCR0_IMODE_OFF 0 -#define UPMCR0_IMODE_AIC 2 -#define UPMCR0_IMODE_HALT 3 -#define UPMCR0_IMODE_FIQ 4 - -/* Uncore PMC status register */ -#define SYS_IMP_APL_UPMSR_EL1 sys_reg(3, 7, 15, 6, 4) -#define UPMSR_IACT BIT(0) - -#define AIC_NR_FIQ 4 -#define AIC_NR_SWIPI 32 - -/* - * FIQ hwirq index definitions: FIQ sources use the DT binding defines - * directly, except that timers are special. At the irqchip level, the - * two timer types are represented by their access method: _EL0 registers - * or _EL02 registers. In the DT binding, the timers are represented - * by their purpose (HV or guest). This mapping is for when the kernel is - * running at EL2 (with VHE). When the kernel is running at EL1, the - * mapping differs and aic_irq_domain_translate() performs the remapping. - */ - -#define AIC_TMR_EL0_PHYS AIC_TMR_HV_PHYS -#define AIC_TMR_EL0_VIRT AIC_TMR_HV_VIRT -#define AIC_TMR_EL02_PHYS AIC_TMR_GUEST_PHYS -#define AIC_TMR_EL02_VIRT AIC_TMR_GUEST_VIRT - struct aic_irq_chip { void __iomem *base; struct irq_domain *hw_domain; struct irq_domain *ipi_domain; int nr_hw; - int ipi_hwirq; }; -static DEFINE_PER_CPU(uint32_t, aic_fiq_unmasked); - -static DEFINE_PER_CPU(atomic_t, aic_vipi_flag); -static DEFINE_PER_CPU(atomic_t, aic_vipi_enable); static struct aic_irq_chip *aic_irqc; @@ -300,153 +217,6 @@ static struct irq_chip aic_chip = { .irq_set_type = aic_irq_set_type, }; -/* - * FIQ irqchip - */ - -static unsigned long aic_fiq_get_idx(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - - return irqd_to_hwirq(d) - ic->nr_hw; -} - -static void aic_fiq_set_mask(struct irq_data *d) -{ - /* Only the guest timers have real mask bits, unfortunately. */ - switch (aic_fiq_get_idx(d)) { - case AIC_TMR_EL02_PHYS: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_P, 0); - isb(); - break; - case AIC_TMR_EL02_VIRT: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_V, 0); - isb(); - break; - default: - break; - } -} - -static void aic_fiq_clear_mask(struct irq_data *d) -{ - switch (aic_fiq_get_idx(d)) { - case AIC_TMR_EL02_PHYS: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_P); - isb(); - break; - case AIC_TMR_EL02_VIRT: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_V); - isb(); - break; - default: - break; - } -} - -static void aic_fiq_mask(struct irq_data *d) -{ - aic_fiq_set_mask(d); - __this_cpu_and(aic_fiq_unmasked, ~BIT(aic_fiq_get_idx(d))); -} - -static void aic_fiq_unmask(struct irq_data *d) -{ - aic_fiq_clear_mask(d); - __this_cpu_or(aic_fiq_unmasked, BIT(aic_fiq_get_idx(d))); -} - -static void aic_fiq_eoi(struct irq_data *d) -{ - /* We mask to ack (where we can), so we need to unmask at EOI. */ - if (__this_cpu_read(aic_fiq_unmasked) & BIT(aic_fiq_get_idx(d))) - aic_fiq_clear_mask(d); -} - -#define TIMER_FIRING(x) \ - (((x) & (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_MASK | \ - ARCH_TIMER_CTRL_IT_STAT)) == \ - (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_STAT)) - -static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) -{ - /* - * It would be really nice if we had a system register that lets us get - * the FIQ source state without having to peek down into sources... - * but such a register does not seem to exist. - * - * So, we have these potential sources to test for: - * - Fast IPIs (not yet used) - * - The 4 timers (CNTP, CNTV for each of HV and guest) - * - Per-core PMCs (not yet supported) - * - Per-cluster uncore PMCs (not yet supported) - * - * Since not dealing with any of these results in a FIQ storm, - * we check for everything here, even things we don't support yet. - */ - - if (read_sysreg_s(SYS_IMP_APL_IPI_SR_EL1) & IPI_SR_PENDING) { - pr_err_ratelimited("Fast IPI fired. Acking.\n"); - write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); - } - - if (TIMER_FIRING(read_sysreg(cntp_ctl_el0))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL0_PHYS, regs); - - if (TIMER_FIRING(read_sysreg(cntv_ctl_el0))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL0_VIRT, regs); - - if (is_kernel_in_hyp_mode()) { - uint64_t enabled = read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2); - - if ((enabled & VM_TMR_FIQ_ENABLE_P) && - TIMER_FIRING(read_sysreg_s(SYS_CNTP_CTL_EL02))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL02_PHYS, regs); - - if ((enabled & VM_TMR_FIQ_ENABLE_V) && - TIMER_FIRING(read_sysreg_s(SYS_CNTV_CTL_EL02))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL02_VIRT, regs); - } - - if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == - (FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_FIQ) | PMCR0_IACT)) { - /* - * Not supported yet, let's figure out how to handle this when - * we implement these proprietary performance counters. For now, - * just mask it and move on. - */ - pr_err_ratelimited("PMC FIQ fired. Masking.\n"); - sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, - FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); - } - - if (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ && - (read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) { - /* Same story with uncore PMCs */ - pr_err_ratelimited("Uncore PMC FIQ fired. Masking.\n"); - sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, - FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); - } -} - -static int aic_fiq_set_type(struct irq_data *d, unsigned int type) -{ - return (type == IRQ_TYPE_LEVEL_HIGH) ? 0 : -EINVAL; -} - -static struct irq_chip fiq_chip = { - .name = "AIC-FIQ", - .irq_mask = aic_fiq_mask, - .irq_unmask = aic_fiq_unmask, - .irq_ack = aic_fiq_set_mask, - .irq_eoi = aic_fiq_eoi, - .irq_set_type = aic_fiq_set_type, -}; - /* * Main IRQ domain */ @@ -485,31 +255,6 @@ static int aic_irq_domain_translate(struct irq_domain *id, return -EINVAL; *hwirq = fwspec->param[1]; break; - case AIC_FIQ: - if (fwspec->param[1] >= AIC_NR_FIQ) - return -EINVAL; - *hwirq = ic->nr_hw + fwspec->param[1]; - - /* - * In EL1 the non-redirected registers are the guest's, - * not EL2's, so remap the hwirqs to match. - */ - if (!is_kernel_in_hyp_mode()) { - switch (fwspec->param[1]) { - case AIC_TMR_GUEST_PHYS: - *hwirq = ic->nr_hw + AIC_TMR_EL0_PHYS; - break; - case AIC_TMR_GUEST_VIRT: - *hwirq = ic->nr_hw + AIC_TMR_EL0_VIRT; - break; - case AIC_TMR_HV_PHYS: - case AIC_TMR_HV_VIRT: - return -ENOENT; - default: - break; - } - } - break; default: return -EINVAL; } @@ -736,33 +481,13 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) static int aic_init_cpu(unsigned int cpu) { - /* Mask all hard-wired per-CPU IRQ/FIQ sources */ - - /* Pending Fast IPI FIQs */ - write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); - - /* Timer FIQs */ - sysreg_clear_set(cntp_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); - sysreg_clear_set(cntv_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); /* EL2-only (VHE mode) IRQ sources */ if (is_kernel_in_hyp_mode()) { - /* Guest timers */ - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, - VM_TMR_FIQ_ENABLE_V | VM_TMR_FIQ_ENABLE_P, 0); - /* vGIC maintenance IRQ */ sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); } - /* PMC FIQ */ - sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, - FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); - - /* Uncore PMC FIQ */ - sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, - FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); - /* Commit all of the above */ isb(); @@ -781,9 +506,6 @@ static int aic_init_cpu(unsigned int cpu) aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_SELF); aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); - /* Initialize the local mask state */ - __this_cpu_write(aic_fiq_unmasked, 0); - return 0; } @@ -833,7 +555,6 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p } set_handle_irq(aic_handle_irq); - set_handle_fiq(aic_handle_fiq); for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) aic_ic_write(irqc, AIC_MASK_SET + i * 4, U32_MAX); @@ -842,9 +563,6 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p for (i = 0; i < irqc->nr_hw; i++) aic_ic_write(irqc, AIC_TARGET_CPU + i * 4, 1); - if (!is_kernel_in_hyp_mode()) - pr_info("Kernel running in EL1, mapping interrupts"); - cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_AIC_STARTING, "irqchip/apple-aic/ipi:starting", aic_init_cpu, NULL); diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c index 13900b4a2ca56e..9229834a04b210 100644 --- a/drivers/irqchip/irq-apple-fiq.c +++ b/drivers/irqchip/irq-apple-fiq.c @@ -9,33 +9,10 @@ */ /* - * AIC is a fairly simple interrupt controller with the following features: - * - * - 896 level-triggered hardware IRQs - * - Single mask bit per IRQ - * - Per-IRQ affinity setting - * - Automatic masking on event delivery (auto-ack) - * - Software triggering (ORed with hw line) - * - 2 per-CPU IPIs (meant as "self" and "other", but they are - * interchangeable if not symmetric) - * - Automatic prioritization (single event/ack register per CPU, lower IRQs = - * higher priority) - * - Automatic masking on ack - * - Default "this CPU" register view and explicit per-CPU views - * - * In addition, this driver also handles FIQs, as these are routed to the same - * IRQ vector. These are used for Fast IPIs (TODO), the ARMv8 timer IRQs, and - * performance counters (TODO). * * Implementation notes: * - * - This driver creates two IRQ domains, one for HW IRQs and internal FIQs, - * and one for IPIs. - * - Since Linux needs more than 2 IPIs, we implement a software IRQ controller - * and funnel all IPIs into one per-CPU IPI (the second "self" IPI is unused). - * - FIQ hwirq numbers are assigned after true hwirqs, and are per-cpu. * - DT bindings use 3-cell form (like GIC): - * - <0 nr flags> - hwirq #nr * - <1 nr flags> - FIQ #nr * - nr=0 Physical HV timer * - nr=1 Virtual HV timer @@ -50,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -61,49 +37,6 @@ #include -/* - * AIC registers (MMIO) - */ - -#define AIC_INFO 0x0004 -#define AIC_INFO_NR_HW GENMASK(15, 0) - -#define AIC_CONFIG 0x0010 - -#define AIC_WHOAMI 0x2000 -#define AIC_EVENT 0x2004 -#define AIC_EVENT_TYPE GENMASK(31, 16) -#define AIC_EVENT_NUM GENMASK(15, 0) - -#define AIC_EVENT_TYPE_HW 1 -#define AIC_EVENT_TYPE_IPI 4 -#define AIC_EVENT_IPI_OTHER 1 -#define AIC_EVENT_IPI_SELF 2 - -#define AIC_IPI_SEND 0x2008 -#define AIC_IPI_ACK 0x200c -#define AIC_IPI_MASK_SET 0x2024 -#define AIC_IPI_MASK_CLR 0x2028 - -#define AIC_IPI_SEND_CPU(cpu) BIT(cpu) - -#define AIC_IPI_OTHER BIT(0) -#define AIC_IPI_SELF BIT(31) - -#define AIC_TARGET_CPU 0x3000 -#define AIC_SW_SET 0x4000 -#define AIC_SW_CLR 0x4080 -#define AIC_MASK_SET 0x4100 -#define AIC_MASK_CLR 0x4180 - -#define AIC_CPU_IPI_SET(cpu) (0x5008 + ((cpu) << 7)) -#define AIC_CPU_IPI_CLR(cpu) (0x500c + ((cpu) << 7)) -#define AIC_CPU_IPI_MASK_SET(cpu) (0x5024 + ((cpu) << 7)) -#define AIC_CPU_IPI_MASK_CLR(cpu) (0x5028 + ((cpu) << 7)) - -#define MASK_REG(x) (4 * ((x) >> 5)) -#define MASK_BIT(x) BIT((x) & GENMASK(4, 0)) - /* * IMP-DEF sysregs that control FIQ sources * Note: sysreg-based IPIs are not supported yet. @@ -173,143 +106,15 @@ #define AIC_TMR_EL02_PHYS AIC_TMR_GUEST_PHYS #define AIC_TMR_EL02_VIRT AIC_TMR_GUEST_VIRT -struct aic_irq_chip { - void __iomem *base; - struct irq_domain *hw_domain; - struct irq_domain *ipi_domain; - int nr_hw; - int ipi_hwirq; -}; - static DEFINE_PER_CPU(uint32_t, aic_fiq_unmasked); -static DEFINE_PER_CPU(atomic_t, aic_vipi_flag); -static DEFINE_PER_CPU(atomic_t, aic_vipi_enable); - static struct aic_irq_chip *aic_irqc; -static void aic_handle_ipi(struct pt_regs *regs); - -static u32 aic_ic_read(struct aic_irq_chip *ic, u32 reg) -{ - return readl_relaxed(ic->base + reg); -} - -static void aic_ic_write(struct aic_irq_chip *ic, u32 reg, u32 val) -{ - writel_relaxed(val, ic->base + reg); -} - -/* - * IRQ irqchip - */ - -static void aic_irq_mask(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - - aic_ic_write(ic, AIC_MASK_SET + MASK_REG(irqd_to_hwirq(d)), - MASK_BIT(irqd_to_hwirq(d))); -} - -static void aic_irq_unmask(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - - aic_ic_write(ic, AIC_MASK_CLR + MASK_REG(irqd_to_hwirq(d)), - MASK_BIT(irqd_to_hwirq(d))); -} - -static void aic_irq_eoi(struct irq_data *d) -{ - /* - * Reading the interrupt reason automatically acknowledges and masks - * the IRQ, so we just unmask it here if needed. - */ - if (!irqd_irq_masked(d)) - aic_irq_unmask(d); -} - -static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) -{ - struct aic_irq_chip *ic = aic_irqc; - u32 event, type, irq; - - do { - /* - * We cannot use a relaxed read here, as reads from DMA buffers - * need to be ordered after the IRQ fires. - */ - event = readl(ic->base + AIC_EVENT); - type = FIELD_GET(AIC_EVENT_TYPE, event); - irq = FIELD_GET(AIC_EVENT_NUM, event); - - if (type == AIC_EVENT_TYPE_HW) - handle_domain_irq(ic->hw_domain, irq, regs); - else if (type == AIC_EVENT_TYPE_IPI) - aic_handle_ipi(0 /* irq */, regs); - else if (event != 0) - pr_err_ratelimited("Unknown IRQ event %d, %d\n", type, irq); - } while (event); - - /* - * vGIC maintenance interrupts end up here too, so we need to check - * for them separately. This should never trigger if KVM is working - * properly, because it will have already taken care of clearing it - * on guest exit before this handler runs. - */ - if (is_kernel_in_hyp_mode() && (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EN) && - read_sysreg_s(SYS_ICH_MISR_EL2) != 0) { - pr_err_ratelimited("vGIC IRQ fired and not handled by KVM, disabling.\n"); - sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); - } -} - -static int aic_irq_set_affinity(struct irq_data *d, - const struct cpumask *mask_val, bool force) -{ - irq_hw_number_t hwirq = irqd_to_hwirq(d); - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - int cpu; - u32 mask = 0; - - for_each_cpu(cpu, mask_val) - mask |= BIT(cpu); - - aic_ic_write(ic, AIC_TARGET_CPU + hwirq * 4, mask); - irq_data_update_effective_affinity(d, mask_val); - - return IRQ_SET_MASK_OK; -} - -static int aic_irq_set_type(struct irq_data *d, unsigned int type) -{ - /* - * Some IRQs (e.g. MSIs) implicitly have edge semantics, and we don't - * have a way to find out the type of any given IRQ, so just allow both. - */ - return (type == IRQ_TYPE_LEVEL_HIGH || type == IRQ_TYPE_EDGE_RISING) ? 0 : -EINVAL; -} - -static struct irq_chip aic_chip = { - .name = "AIC", - .irq_mask = aic_irq_mask, - .irq_unmask = aic_irq_unmask, - .irq_eoi = aic_irq_eoi, - .irq_set_affinity = aic_irq_set_affinity, - .irq_set_type = aic_irq_set_type, -}; /* * FIQ irqchip */ -static unsigned long aic_fiq_get_idx(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - - return irqd_to_hwirq(d) - ic->nr_hw; -} static void aic_fiq_set_mask(struct irq_data *d) { @@ -540,148 +345,7 @@ static int aic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, return 0; } -static void aic_irq_domain_free(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs) -{ - int i; - - for (i = 0; i < nr_irqs; i++) { - struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); - - irq_set_handler(virq + i, NULL); - irq_domain_reset_irq_data(d); - } -} - -static const struct irq_domain_ops aic_irq_domain_ops = { - .translate = aic_irq_domain_translate, - .alloc = aic_irq_domain_alloc, - .free = aic_irq_domain_free, -}; - -/* - * IPI irqchip - */ - -static void aic_ipi_mask(struct irq_data *d) { - u32 irq_bit = BIT(irqd_to_hwirq(d)); - - /* No specific ordering requirements needed here. */ - atomic_andnot(irq_bit, this_cpu_ptr(&aic_vipi_enable)); -} - -static void aic_ipi_unmask(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - u32 irq_bit = BIT(irqd_to_hwirq(d)); - - atomic_or(irq_bit, this_cpu_ptr(&aic_vipi_enable)); - - /* - * The atomic_or() above must complete before the atomic_read() - * below to avoid racing aic_ipi_send_mask(). - */ - smp_mb__after_atomic(); - - /* - * If a pending vIPI was unmasked, raise a HW IPI to ourselves. - * No barriers needed here since this is a self-IPI. - */ - if (atomic_read(this_cpu_ptr(&aic_vipi_flag)) & irq_bit) - aic_ic_write(ic, AIC_IPI_SEND, AIC_IPI_SEND_CPU(smp_processor_id())); -} - -static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - u32 irq_bit = BIT(irqd_to_hwirq(d)); - u32 send = 0; - int cpu; - unsigned long pending; - - for_each_cpu(cpu, mask) { - /* - * This sequence is the mirror of the one in aic_ipi_unmask(); - * see the comment there. Additionally, release semantics - * ensure that the vIPI flag set is ordered after any shared - * memory accesses that precede it. This therefore also pairs - * with the atomic_fetch_andnot in aic_handle_ipi(). - */ - pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&aic_vipi_flag, cpu)); - - /* - * The atomic_fetch_or_release() above must complete before the - * atomic_read() below to avoid racing aic_ipi_unmask(). - */ - smp_mb__after_atomic(); - - if (!(pending & irq_bit) && - (atomic_read(per_cpu_ptr(&aic_vipi_enable, cpu)) & irq_bit)) - send |= AIC_IPI_SEND_CPU(cpu); - } - - /* - * The flag writes must complete before the physical IPI is issued - * to another CPU. This is implied by the control dependency on - * the result of atomic_read_acquire() above, which is itself - * already ordered after the vIPI flag write. - */ - if (send) - aic_ic_write(ic, AIC_IPI_SEND, send); -} - -static struct irq_chip ipi_chip = { - .name = "AIC-IPI", - .irq_mask = aic_ipi_mask, - .irq_unmask = aic_ipi_unmask, - .ipi_send_mask = aic_ipi_send_mask, -}; - -/* - * IPI IRQ domain - */ - -static void aic_handle_ipi(struct pt_regs *regs) -{ - int i; - unsigned long enabled, firing; - - /* - * Ack the IPI. We need to order this after the AIC event read, but - * that is enforced by normal MMIO ordering guarantees. - */ - aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_OTHER); - - /* - * The mask read does not need to be ordered. Only we can change - * our own mask anyway, so no races are possible here, as long as - * we are properly in the interrupt handler (which is covered by - * the barrier that is part of the top-level AIC handler's readl()). - */ - enabled = atomic_read(this_cpu_ptr(&aic_vipi_enable)); - - /* - * Clear the IPIs we are about to handle. This pairs with the - * atomic_fetch_or_release() in aic_ipi_send_mask(), and needs to be - * ordered after the aic_ic_write() above (to avoid dropping vIPIs) and - * before IPI handling code (to avoid races handling vIPIs before they - * are signaled). The former is taken care of by the release semantics - * of the write portion, while the latter is taken care of by the - * acquire semantics of the read portion. - */ - firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&aic_vipi_flag)) & enabled; - - for_each_set_bit(i, &firing, AIC_NR_SWIPI) - handle_domain_irq(aic_irqc->ipi_domain, i, regs); - - /* - * No ordering needed here; at worst this just changes the timing of - * when the next IPI will be delivered. - */ - aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); -} - static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs, void *args) { @@ -696,16 +360,6 @@ static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, return 0; } -static void aic_ipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) -{ - /* Not freeing IPIs */ -} - -static const struct irq_domain_ops aic_ipi_domain_ops = { - .alloc = aic_ipi_alloc, - .free = aic_ipi_free, -}; - static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) { struct irq_domain *ipi_domain; @@ -765,9 +419,6 @@ static int aic_init_cpu(unsigned int cpu) /* Guest timers */ sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_V | VM_TMR_FIQ_ENABLE_P, 0); - - /* vGIC maintenance IRQ */ - sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); } /* PMC FIQ */ @@ -781,32 +432,12 @@ static int aic_init_cpu(unsigned int cpu) /* Commit all of the above */ isb(); - /* - * Make sure the kernel's idea of logical CPU order is the same as AIC's - * If we ever end up with a mismatch here, we will have to introduce - * a mapping table similar to what other irqchip drivers do. - */ - WARN_ON(aic_ic_read(aic_irqc, AIC_WHOAMI) != smp_processor_id()); - - /* - * Always keep IPIs unmasked at the hardware level (except auto-masking - * by AIC during processing). We manage masks at the vIPI level. - */ - aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_SELF | AIC_IPI_OTHER); - aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_SELF); - aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); - /* Initialize the local mask state */ __this_cpu_write(aic_fiq_unmasked, 0); return 0; } -static struct gic_kvm_info vgic_info __initdata = { - .type = GIC_V3, - .no_maint_irq_mask = true, - .no_hw_deactivation = true, -}; static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) { @@ -847,15 +478,6 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p return -ENODEV; } - set_handle_irq(aic_handle_irq); - set_handle_fiq(aic_handle_fiq); - - for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) - aic_ic_write(irqc, AIC_MASK_SET + i * 4, U32_MAX); - for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) - aic_ic_write(irqc, AIC_SW_CLR + i * 4, U32_MAX); - for (i = 0; i < irqc->nr_hw; i++) - aic_ic_write(irqc, AIC_TARGET_CPU + i * 4, 1); if (!is_kernel_in_hyp_mode()) pr_info("Kernel running in EL1, mapping interrupts"); From a00e31258b2378f011709a233a2ec2bb9b0fca60 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 20:16:54 +0000 Subject: [PATCH 11/16] irq-apple-aic: simplify by removing FIQ-specific code Signed-off-by: Pip Cet --- drivers/irqchip/irq-apple-aic.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index ce7bbd3ade23b0..908e018a7d386a 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -224,17 +224,9 @@ static struct irq_chip aic_chip = { static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, irq_hw_number_t hw) { - struct aic_irq_chip *ic = id->host_data; - - if (hw < ic->nr_hw) { - irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, - handle_fasteoi_irq, NULL, NULL); - irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); - } else { - irq_set_percpu_devid(irq); - irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, - handle_percpu_devid_irq, NULL, NULL); - } + irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, + handle_fasteoi_irq, NULL, NULL); + irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); return 0; } @@ -481,6 +473,7 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) static int aic_init_cpu(unsigned int cpu) { + /* Mask hard-wired per-CPU IRQ sources */ /* EL2-only (VHE mode) IRQ sources */ if (is_kernel_in_hyp_mode()) { @@ -488,7 +481,7 @@ static int aic_init_cpu(unsigned int cpu) sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); } - /* Commit all of the above */ + /* Commit the above */ isb(); /* @@ -537,7 +530,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p irqc->nr_hw = FIELD_GET(AIC_INFO_NR_HW, info); irqc->hw_domain = irq_domain_create_linear(of_node_to_fwnode(node), - irqc->nr_hw + AIC_NR_FIQ, + irqc->nr_hw, &aic_irq_domain_ops, irqc); if (WARN_ON(!irqc->hw_domain)) { iounmap(irqc->base); From 4c647839227dc1058762d3eb3a0b27632d964697 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 18:42:07 +0000 Subject: [PATCH 12/16] irq-apple-aic: simplify IPI handling by relying on external vIPI In preparation to switching to "fast IPIs" on most Apple M1 systems, code is moved out of irq-apple-aic and the surviving code, simplified. IPI use is now optional and depends on a DT property. Signed-off-by: Pip Cet --- drivers/irqchip/irq-apple-aic.c | 114 ++++++++------------------------ 1 file changed, 28 insertions(+), 86 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 908e018a7d386a..7e242ba9d1dcee 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -25,11 +25,11 @@ * * Implementation notes: * - * - This driver creates two IRQ domains, one for HW IRQs and internal FIQs, - * and one for IPIs. - * - Since Linux needs more than 2 IPIs, we implement a software IRQ controller - * and funnel all IPIs into one per-CPU IPI (the second "self" IPI is unused). - * - FIQ hwirq numbers are assigned after true hwirqs, and are per-cpu. + * - This driver creates two IRQ domains, one for HW IRQs, and one for + * the single IPI we actually support. + * - Since Linux needs more than 2 IPIs, we rely on the arch IRQ layer + * to funnel IPIs through its own implementation, using just one + * per-CPU real IPI (the second "self" IPI is unused). * - DT bindings use 3-cell form (like GIC): * - <0 nr flags> - hwirq #nr */ @@ -102,10 +102,11 @@ struct aic_irq_chip { int nr_hw; }; +#define AIC_NR_IPI 1 static struct aic_irq_chip *aic_irqc; -static void aic_handle_ipi(struct pt_regs *regs); +static void aic_handle_ipi(int index, struct pt_regs *regs); static u32 aic_ic_read(struct aic_irq_chip *ic, u32 reg) { @@ -300,70 +301,30 @@ static const struct irq_domain_ops aic_irq_domain_ops = { * IPI irqchip */ -static void aic_ipi_mask(struct irq_data *d) +static int aic_ipi_number(struct irq_data *d) { - u32 irq_bit = BIT(irqd_to_hwirq(d)); + return irqd_to_hwirq(d) ? AIC_IPI_OTHER : AIC_IPI_OTHER; +} - /* No specific ordering requirements needed here. */ - atomic_andnot(irq_bit, this_cpu_ptr(&aic_vipi_enable)); +static void aic_ipi_mask(struct irq_data *d) +{ + aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, aic_ipi_number(d)); } static void aic_ipi_unmask(struct irq_data *d) { - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - u32 irq_bit = BIT(irqd_to_hwirq(d)); - - atomic_or(irq_bit, this_cpu_ptr(&aic_vipi_enable)); - - /* - * The atomic_or() above must complete before the atomic_read() - * below to avoid racing aic_ipi_send_mask(). - */ - smp_mb__after_atomic(); - - /* - * If a pending vIPI was unmasked, raise a HW IPI to ourselves. - * No barriers needed here since this is a self-IPI. - */ - if (atomic_read(this_cpu_ptr(&aic_vipi_flag)) & irq_bit) - aic_ic_write(ic, AIC_IPI_SEND, AIC_IPI_SEND_CPU(smp_processor_id())); + aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, aic_ipi_number(d)); } static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) { struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - u32 irq_bit = BIT(irqd_to_hwirq(d)); u32 send = 0; int cpu; - unsigned long pending; - for_each_cpu(cpu, mask) { - /* - * This sequence is the mirror of the one in aic_ipi_unmask(); - * see the comment there. Additionally, release semantics - * ensure that the vIPI flag set is ordered after any shared - * memory accesses that precede it. This therefore also pairs - * with the atomic_fetch_andnot in aic_handle_ipi(). - */ - pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&aic_vipi_flag, cpu)); + for_each_cpu(cpu, mask) + send |= AIC_IPI_SEND_CPU(cpu); - /* - * The atomic_fetch_or_release() above must complete before the - * atomic_read() below to avoid racing aic_ipi_unmask(). - */ - smp_mb__after_atomic(); - - if (!(pending & irq_bit) && - (atomic_read(per_cpu_ptr(&aic_vipi_enable, cpu)) & irq_bit)) - send |= AIC_IPI_SEND_CPU(cpu); - } - - /* - * The flag writes must complete before the physical IPI is issued - * to another CPU. This is implied by the control dependency on - * the result of atomic_read_acquire() above, which is itself - * already ordered after the vIPI flag write. - */ if (send) aic_ic_write(ic, AIC_IPI_SEND, send); } @@ -379,44 +340,24 @@ static struct irq_chip ipi_chip = { * IPI IRQ domain */ -static void aic_handle_ipi(struct pt_regs *regs) +static void aic_handle_ipi(int index, struct pt_regs *regs) { - int i; - unsigned long enabled, firing; - + struct irq_domain *domain = aic_irqc->ipi_domain; + struct aic_irq_chip *ic = aic_irqc; /* * Ack the IPI. We need to order this after the AIC event read, but * that is enforced by normal MMIO ordering guarantees. */ - aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_OTHER); + aic_ic_write(ic, AIC_IPI_ACK, + aic_ipi_number(irq_domain_get_irq_data(domain, index))); - /* - * The mask read does not need to be ordered. Only we can change - * our own mask anyway, so no races are possible here, as long as - * we are properly in the interrupt handler (which is covered by - * the barrier that is part of the top-level AIC handler's readl()). - */ - enabled = atomic_read(this_cpu_ptr(&aic_vipi_enable)); - - /* - * Clear the IPIs we are about to handle. This pairs with the - * atomic_fetch_or_release() in aic_ipi_send_mask(), and needs to be - * ordered after the aic_ic_write() above (to avoid dropping vIPIs) and - * before IPI handling code (to avoid races handling vIPIs before they - * are signaled). The former is taken care of by the release semantics - * of the write portion, while the latter is taken care of by the - * acquire semantics of the read portion. - */ - firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&aic_vipi_flag)) & enabled; - - for_each_set_bit(i, &firing, AIC_NR_SWIPI) - handle_domain_irq(aic_irqc->ipi_domain, i, regs); + handle_domain_irq(domain, index, regs); /* * No ordering needed here; at worst this just changes the timing of * when the next IPI will be delivered. */ - aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); + aic_ic_write(ic, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); } static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, @@ -448,7 +389,7 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) struct irq_domain *ipi_domain; int base_ipi; - ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_SWIPI, + ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_IPI, &aic_ipi_domain_ops, irqc); if (WARN_ON(!ipi_domain)) return -ENODEV; @@ -456,7 +397,7 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); - base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI, + base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_IPI, NUMA_NO_NODE, NULL, false, NULL); if (WARN_ON(base_ipi < 0)) { @@ -464,7 +405,7 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) return -ENODEV; } - set_smp_ipi_range(base_ipi, AIC_NR_SWIPI); + set_smp_ipi_range(base_ipi, AIC_NR_IPI); irqc->ipi_domain = ipi_domain; @@ -514,6 +455,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p void __iomem *regs; u32 info; struct aic_irq_chip *irqc; + bool use_for_ipi = of_property_read_bool(node, "use-for-ipi"); regs = of_iomap(node, 0); if (WARN_ON(!regs)) @@ -540,7 +482,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p irq_domain_update_bus_token(irqc->hw_domain, DOMAIN_BUS_WIRED); - if (aic_init_smp(irqc, node)) { + if (use_for_ipi && aic_init_smp(irqc, node)) { irq_domain_remove(irqc->hw_domain); iounmap(irqc->base); kfree(irqc); From ea9e0fa3d7f9843b69f74a6c3df1d5baba6e999f Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 18:45:01 +0000 Subject: [PATCH 13/16] irq-apple-fiq: build along with irq-apple-aic They currently make no sense without each other, but that might change soon. Signed-off-by: Pip Cet --- drivers/irqchip/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index f88cbf36a9d28d..3d4e246b7e4547 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -115,4 +115,4 @@ obj-$(CONFIG_SL28CPLD_INTC) += irq-sl28cpld.o obj-$(CONFIG_MACH_REALTEK_RTL) += irq-realtek-rtl.o obj-$(CONFIG_WPCM450_AIC) += irq-wpcm450-aic.o obj-$(CONFIG_IRQ_IDT3243X) += irq-idt3243x.o -obj-$(CONFIG_APPLE_AIC) += irq-apple-aic.o +obj-$(CONFIG_APPLE_AIC) += irq-apple-aic.o irq-apple-fiq.o From 1d4c7178eeccd0b23df090d83ebb9184c4b8f99a Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 19:54:43 +0000 Subject: [PATCH 14/16] irq-apple-fiq: mechanical replacement of aic_ prefixes No functional changes intended. Signed-off-by: Pip Cet --- drivers/irqchip/irq-apple-fiq.c | 63 +++++++++++++++++---------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c index 9229834a04b210..30f6125e845fbe 100644 --- a/drivers/irqchip/irq-apple-fiq.c +++ b/drivers/irqchip/irq-apple-fiq.c @@ -88,8 +88,8 @@ #define SYS_IMP_APL_UPMSR_EL1 sys_reg(3, 7, 15, 6, 4) #define UPMSR_IACT BIT(0) -#define AIC_NR_FIQ 4 -#define AIC_NR_SWIPI 32 +#define NR_FIQ 6 +#define FIQ_NR_IPI 1 /* * FIQ hwirq index definitions: FIQ sources use the DT binding defines @@ -98,7 +98,7 @@ * or _EL02 registers. In the DT binding, the timers are represented * by their purpose (HV or guest). This mapping is for when the kernel is * running at EL2 (with VHE). When the kernel is running at EL1, the - * mapping differs and aic_irq_domain_translate() performs the remapping. + * mapping differs and irq_domain_translate() performs the remapping. */ #define AIC_TMR_EL0_PHYS AIC_TMR_HV_PHYS @@ -106,17 +106,18 @@ #define AIC_TMR_EL02_PHYS AIC_TMR_GUEST_PHYS #define AIC_TMR_EL02_VIRT AIC_TMR_GUEST_VIRT -static DEFINE_PER_CPU(uint32_t, aic_fiq_unmasked); -static struct aic_irq_chip *aic_irqc; +static DEFINE_PER_CPU(uint32_t, fiq_unmasked); + +static struct fiq_irq_chip *fiq_irqc; /* * FIQ irqchip */ -static void aic_fiq_set_mask(struct irq_data *d) +static void fiq_set_mask(struct irq_data *d) { /* Only the guest timers have real mask bits, unfortunately. */ switch (aic_fiq_get_idx(d)) { @@ -133,14 +134,14 @@ static void aic_fiq_set_mask(struct irq_data *d) } } -static void aic_fiq_clear_mask(struct irq_data *d) +static void fiq_clear_mask(struct irq_data *d) { switch (aic_fiq_get_idx(d)) { case AIC_TMR_EL02_PHYS: sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_P); isb(); break; - case AIC_TMR_EL02_VIRT: + case FIQ_TMR_EL02_VIRT: sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_V); isb(); break; @@ -173,7 +174,7 @@ static void aic_fiq_eoi(struct irq_data *d) ARCH_TIMER_CTRL_IT_STAT)) == \ (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_STAT)) -static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) +static void __exception_irq_entry handle_fiq(struct pt_regs *regs) { /* * It would be really nice if we had a system register that lets us get @@ -238,26 +239,26 @@ static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) } } -static int aic_fiq_set_type(struct irq_data *d, unsigned int type) +static int fiq_set_type(struct irq_data *d, unsigned int type) { return (type == IRQ_TYPE_LEVEL_HIGH) ? 0 : -EINVAL; } static struct irq_chip fiq_chip = { - .name = "AIC-FIQ", - .irq_mask = aic_fiq_mask, - .irq_unmask = aic_fiq_unmask, - .irq_ack = aic_fiq_set_mask, - .irq_eoi = aic_fiq_eoi, - .irq_set_type = aic_fiq_set_type, + .name = "FIQ", + .irq_mask = fiq_mask, + .irq_unmask = fiq_unmask, + .irq_ack = fiq_set_mask, + .irq_eoi = fiq_eoi, + .irq_set_type = fiq_set_type, }; /* * Main IRQ domain */ -static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, - irq_hw_number_t hw) +static int irq_domain_map(struct irq_domain *id, unsigned int irq, + irq_hw_number_t hw) { struct aic_irq_chip *ic = id->host_data; @@ -274,10 +275,10 @@ static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, return 0; } -static int aic_irq_domain_translate(struct irq_domain *id, - struct irq_fwspec *fwspec, - unsigned long *hwirq, - unsigned int *type) +static int irq_domain_translate(struct irq_domain *id, + struct irq_fwspec *fwspec, + unsigned long *hwirq, + unsigned int *type) { struct aic_irq_chip *ic = id->host_data; @@ -307,8 +308,8 @@ static int aic_irq_domain_translate(struct irq_domain *id, case AIC_TMR_GUEST_VIRT: *hwirq = ic->nr_hw + AIC_TMR_EL0_VIRT; break; - case AIC_TMR_HV_PHYS: - case AIC_TMR_HV_VIRT: + case FIQ_TMR_HV_PHYS: + case FIQ_TMR_HV_VIRT: return -ENOENT; default: break; @@ -324,20 +325,20 @@ static int aic_irq_domain_translate(struct irq_domain *id, return 0; } -static int aic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs, void *arg) +static int irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) { unsigned int type = IRQ_TYPE_NONE; struct irq_fwspec *fwspec = arg; irq_hw_number_t hwirq; int i, ret; - ret = aic_irq_domain_translate(domain, fwspec, &hwirq, &type); + ret = irq_domain_translate(domain, fwspec, &hwirq, &type); if (ret) return ret; for (i = 0; i < nr_irqs; i++) { - ret = aic_irq_domain_map(domain, virq + i, hwirq + i); + ret = irq_domain_map(domain, virq + i, hwirq + i); if (ret) return ret; } @@ -403,7 +404,7 @@ static const struct irq_domain_ops ipi_domain_ops = { .free = irq_domain_free, }; -static int aic_init_cpu(unsigned int cpu) +static int fiq_init_cpu(unsigned int cpu) { /* Mask all hard-wired per-CPU IRQ/FIQ sources */ @@ -433,7 +434,7 @@ static int aic_init_cpu(unsigned int cpu) isb(); /* Initialize the local mask state */ - __this_cpu_write(aic_fiq_unmasked, 0); + __this_cpu_write(fiq_unmasked, 0); return 0; } @@ -494,4 +495,4 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p return 0; } -IRQCHIP_DECLARE(apple_m1_aic, "apple,aic", aic_of_ic_init); +IRQCHIP_DECLARE(apple_m1_fiq, "apple,fiq", fiq_of_ic_init); From 44401c9ad669cb9e5443a73a7b504d253dec0a8c Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 18:56:51 +0000 Subject: [PATCH 15/16] irq-apple-fiq: FIQ code adjustments In Apple M1 ARM64 SoCs, many interrupts are FIQs (fast IRQs) rather than "real" IRQs. Those used to be handled by the IRQ driver, but it makes more sense to have a separate driver as there are no interdependencies. Like the IRQ driver, irq-apple-fiq relies on an external vIPI layer since it doesn't provide a sufficient number of IPIs. A cpuhotplug state was added so CPUs don't come online with pending FIQs, which would crash them as soon as the F flag in the PSTATE word is cleared. Signed-off-by: Pip Cet --- drivers/irqchip/irq-apple-aic.c | 6 +- drivers/irqchip/irq-apple-fiq.c | 277 +++++++++++++++++++------------- include/linux/cpuhotplug.h | 1 + 3 files changed, 168 insertions(+), 116 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 7e242ba9d1dcee..8df10e1fc2ed4e 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -175,6 +175,8 @@ static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) * for them separately. This should never trigger if KVM is working * properly, because it will have already taken care of clearing it * on guest exit before this handler runs. + * + * XXX it would be nice to skip this check. */ if (is_kernel_in_hyp_mode() && (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EN) && read_sysreg_s(SYS_ICH_MISR_EL2) != 0) { @@ -504,8 +506,8 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p vgic_set_kvm_info(&vgic_info); - pr_info("Initialized with %d IRQs, %d FIQs, %d vIPIs\n", - irqc->nr_hw, AIC_NR_FIQ, AIC_NR_SWIPI); + pr_info("Initialized with %d IRQs, 1 IPI, %sused for IPI\n", irqc->nr_hw, + use_for_ipi ? "" : "not "); return 0; } diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c index 30f6125e845fbe..abd1d9924d9a11 100644 --- a/drivers/irqchip/irq-apple-fiq.c +++ b/drivers/irqchip/irq-apple-fiq.c @@ -9,9 +9,13 @@ */ /* + * This driver handles FIQs. These are used for Fast IPIs, the ARMv8 + * timer IRQs, and performance counters (TODO). * * Implementation notes: * + * - This driver creates one IRQ domain for FIQs and another for the + * single IPI that is used. * - DT bindings use 3-cell form (like GIC): * - <1 nr flags> - FIQ #nr * - nr=0 Physical HV timer @@ -101,31 +105,39 @@ * mapping differs and irq_domain_translate() performs the remapping. */ -#define AIC_TMR_EL0_PHYS AIC_TMR_HV_PHYS -#define AIC_TMR_EL0_VIRT AIC_TMR_HV_VIRT -#define AIC_TMR_EL02_PHYS AIC_TMR_GUEST_PHYS -#define AIC_TMR_EL02_VIRT AIC_TMR_GUEST_VIRT +#define FIQ_TMR_HV_PHYS AIC_TMR_HV_PHYS +#define FIQ_TMR_HV_VIRT AIC_TMR_HV_VIRT +#define FIQ_TMR_GUEST_PHYS AIC_TMR_GUEST_PHYS +#define FIQ_TMR_GUEST_VIRT AIC_TMR_GUEST_VIRT +#define FIQ_TMR_EL0_PHYS FIQ_TMR_HV_PHYS +#define FIQ_TMR_EL0_VIRT FIQ_TMR_HV_VIRT +#define FIQ_TMR_EL02_PHYS FIQ_TMR_GUEST_PHYS +#define FIQ_TMR_EL02_VIRT FIQ_TMR_GUEST_VIRT static DEFINE_PER_CPU(uint32_t, fiq_unmasked); -static struct fiq_irq_chip *fiq_irqc; - /* * FIQ irqchip */ +struct fiq_irq_chip { + struct irq_domain *domain; + struct irq_domain *ipi_domain; +}; + +static struct fiq_irq_chip *fiq_irqc; static void fiq_set_mask(struct irq_data *d) { /* Only the guest timers have real mask bits, unfortunately. */ - switch (aic_fiq_get_idx(d)) { - case AIC_TMR_EL02_PHYS: + switch (irqd_to_hwirq(d)) { + case FIQ_TMR_EL02_PHYS: sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_P, 0); isb(); break; - case AIC_TMR_EL02_VIRT: + case FIQ_TMR_EL02_VIRT: sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_V, 0); isb(); break; @@ -136,8 +148,8 @@ static void fiq_set_mask(struct irq_data *d) static void fiq_clear_mask(struct irq_data *d) { - switch (aic_fiq_get_idx(d)) { - case AIC_TMR_EL02_PHYS: + switch (irqd_to_hwirq(d)) { + case FIQ_TMR_EL02_PHYS: sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_P); isb(); break; @@ -150,23 +162,37 @@ static void fiq_clear_mask(struct irq_data *d) } } -static void aic_fiq_mask(struct irq_data *d) +static void fiq_mask(struct irq_data *d) { - aic_fiq_set_mask(d); - __this_cpu_and(aic_fiq_unmasked, ~BIT(aic_fiq_get_idx(d))); + fiq_set_mask(d); + __this_cpu_and(fiq_unmasked, ~BIT(irqd_to_hwirq(d))); } -static void aic_fiq_unmask(struct irq_data *d) +static void fiq_unmask(struct irq_data *d) { - aic_fiq_clear_mask(d); - __this_cpu_or(aic_fiq_unmasked, BIT(aic_fiq_get_idx(d))); + fiq_clear_mask(d); + __this_cpu_or(fiq_unmasked, BIT(irqd_to_hwirq(d))); } -static void aic_fiq_eoi(struct irq_data *d) +static void fiq_eoi(struct irq_data *d) { /* We mask to ack (where we can), so we need to unmask at EOI. */ - if (__this_cpu_read(aic_fiq_unmasked) & BIT(aic_fiq_get_idx(d))) - aic_fiq_clear_mask(d); + if (__this_cpu_read(fiq_unmasked) & BIT(irqd_to_hwirq(d))) + fiq_clear_mask(d); +} + +static void fiq_ipi_mask(struct irq_data *d) +{ + /* The vIPI layer does not assume IPIs are masked. */ +} + +static void fiq_ipi_unmask(struct irq_data *d) +{ + /* The vIPI layer does not assume IPIs are masked. */ +} + +static void fiq_ipi_eoi(struct irq_data *d) +{ } #define TIMER_FIRING(x) \ @@ -176,6 +202,7 @@ static void aic_fiq_eoi(struct irq_data *d) static void __exception_irq_entry handle_fiq(struct pt_regs *regs) { + struct fiq_irq_chip *ic = fiq_irqc; /* * It would be really nice if we had a system register that lets us get * the FIQ source state without having to peek down into sources... @@ -192,30 +219,25 @@ static void __exception_irq_entry handle_fiq(struct pt_regs *regs) */ if (read_sysreg_s(SYS_IMP_APL_IPI_SR_EL1) & IPI_SR_PENDING) { - pr_err_ratelimited("Fast IPI fired. Acking.\n"); write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); + handle_domain_irq(ic->ipi_domain, 0, regs); } if (TIMER_FIRING(read_sysreg(cntp_ctl_el0))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL0_PHYS, regs); + handle_domain_irq(ic->domain, FIQ_TMR_EL0_PHYS, regs); if (TIMER_FIRING(read_sysreg(cntv_ctl_el0))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL0_VIRT, regs); - + handle_domain_irq(ic->domain, FIQ_TMR_EL0_VIRT, regs); if (is_kernel_in_hyp_mode()) { uint64_t enabled = read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2); if ((enabled & VM_TMR_FIQ_ENABLE_P) && TIMER_FIRING(read_sysreg_s(SYS_CNTP_CTL_EL02))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL02_PHYS, regs); + handle_domain_irq(ic->domain, FIQ_TMR_EL02_PHYS, regs); if ((enabled & VM_TMR_FIQ_ENABLE_V) && TIMER_FIRING(read_sysreg_s(SYS_CNTV_CTL_EL02))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL02_VIRT, regs); + handle_domain_irq(fiq_irqc->domain, FIQ_TMR_EL02_VIRT, regs); } if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == @@ -244,6 +266,29 @@ static int fiq_set_type(struct irq_data *d, unsigned int type) return (type == IRQ_TYPE_LEVEL_HIGH) ? 0 : -EINVAL; } +static void fiq_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) +{ + int cpu; + + for_each_cpu(cpu, mask) { + int lcpu = get_cpu(); + + if ((lcpu ^ cpu) & 4) { + u64 val = (FIELD_PREP(IPI_RR_TYPE, IPI_RR_IMMEDIATE) | + FIELD_PREP(IPI_RR_CLUSTER, !!(cpu & 4)) | + (cpu & 3)); + write_sysreg_s(val, SYS_IMP_APL_IPI_RR_GLOBAL_EL1); + } else { + u64 val = cpu & 3; + write_sysreg_s(val, SYS_IMP_APL_IPI_RR_LOCAL_EL1); + } + + put_cpu(); + } + + isb(); +} + static struct irq_chip fiq_chip = { .name = "FIQ", .irq_mask = fiq_mask, @@ -253,6 +298,13 @@ static struct irq_chip fiq_chip = { .irq_set_type = fiq_set_type, }; +static struct irq_chip fiq_ipi_chip = { + .name = "FIQ-IPI", + .irq_mask = fiq_ipi_mask, + .irq_unmask = fiq_ipi_unmask, + .ipi_send_mask = fiq_ipi_send_mask, +}; + /* * Main IRQ domain */ @@ -260,17 +312,9 @@ static struct irq_chip fiq_chip = { static int irq_domain_map(struct irq_domain *id, unsigned int irq, irq_hw_number_t hw) { - struct aic_irq_chip *ic = id->host_data; - - if (hw < ic->nr_hw) { - irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, - handle_fasteoi_irq, NULL, NULL); - irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); - } else { - irq_set_percpu_devid(irq); - irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, - handle_percpu_devid_irq, NULL, NULL); - } + irq_set_percpu_devid(irq); + irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, + handle_percpu_devid_irq, NULL, NULL); return 0; } @@ -280,21 +324,14 @@ static int irq_domain_translate(struct irq_domain *id, unsigned long *hwirq, unsigned int *type) { - struct aic_irq_chip *ic = id->host_data; - if (fwspec->param_count != 3 || !is_of_node(fwspec->fwnode)) return -EINVAL; switch (fwspec->param[0]) { - case AIC_IRQ: - if (fwspec->param[1] >= ic->nr_hw) - return -EINVAL; - *hwirq = fwspec->param[1]; - break; case AIC_FIQ: - if (fwspec->param[1] >= AIC_NR_FIQ) + if (fwspec->param[1] >= NR_FIQ) return -EINVAL; - *hwirq = ic->nr_hw + fwspec->param[1]; + *hwirq = fwspec->param[1]; /* * In EL1 the non-redirected registers are the guest's, @@ -302,11 +339,11 @@ static int irq_domain_translate(struct irq_domain *id, */ if (!is_kernel_in_hyp_mode()) { switch (fwspec->param[1]) { - case AIC_TMR_GUEST_PHYS: - *hwirq = ic->nr_hw + AIC_TMR_EL0_PHYS; + case FIQ_TMR_GUEST_PHYS: + *hwirq = FIQ_TMR_EL0_PHYS; break; - case AIC_TMR_GUEST_VIRT: - *hwirq = ic->nr_hw + AIC_TMR_EL0_VIRT; + case FIQ_TMR_GUEST_VIRT: + *hwirq = FIQ_TMR_EL0_VIRT; break; case FIQ_TMR_HV_PHYS: case FIQ_TMR_HV_VIRT: @@ -345,41 +382,22 @@ static int irq_domain_alloc(struct irq_domain *domain, unsigned int virq, return 0; } - -{ -static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, - unsigned int nr_irqs, void *args) +static int ipi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) { - int i; + unsigned int type = IRQ_TYPE_NONE; + irq_hw_number_t hwirq; + int i, ret; for (i = 0; i < nr_irqs; i++) { irq_set_percpu_devid(virq + i); - irq_domain_set_info(d, virq + i, i, &ipi_chip, d->host_data, + irq_domain_set_info(domain, virq + i, i, &fiq_ipi_chip, domain->host_data, handle_percpu_devid_irq, NULL, NULL); } return 0; } -static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) -{ - struct irq_domain *ipi_domain; - int base_ipi; - - ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_SWIPI, - &aic_ipi_domain_ops, irqc); - if (WARN_ON(!ipi_domain)) - return -ENODEV; - - ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; - irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); - - base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI, - NUMA_NO_NODE, NULL, false, NULL); - - if (WARN_ON(base_ipi < 0)) { - irq_domain_remove(ipi_domain); - return -ENODEV; static void irq_domain_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs) { @@ -406,7 +424,7 @@ static const struct irq_domain_ops ipi_domain_ops = { static int fiq_init_cpu(unsigned int cpu) { - /* Mask all hard-wired per-CPU IRQ/FIQ sources */ + /* Mask all hard-wired per-CPU FIQ sources */ /* Pending Fast IPI FIQs */ write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); @@ -439,58 +457,89 @@ static int fiq_init_cpu(unsigned int cpu) return 0; } +/* Regular old IRQ handler for "other" FIQs. This will have to go away + * and forward the PMC FIQs at some point, but for now it's better to + * have the stats that we get from a regular IRQ handler. */ -static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) +static enum irqreturn fiq_handler(int irq, void *ptr) { - int i; - void __iomem *regs; - u32 info; - struct aic_irq_chip *irqc; + if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == + (FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_FIQ) | PMCR0_IACT)) { + /* + * Not supported yet, let's figure out how to handle this when + * we implement these proprietary performance counters. For now, + * just mask it and move on. + */ + pr_err_ratelimited("PMC FIQ fired. Masking.\n"); + sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, + FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); + return IRQ_HANDLED; + } else if (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ && + (read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) { + /* Same story with uncore PMCs */ + pr_err_ratelimited("Uncore PMC FIQ fired. Masking.\n"); + sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, + FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); + return IRQ_HANDLED; + } - regs = of_iomap(node, 0); - if (WARN_ON(!regs)) - return -EIO; + return IRQ_NONE; +} - irqc = kzalloc(sizeof(*irqc), GFP_KERNEL); - if (!irqc) - return -ENOMEM; +static int __init fiq_of_ic_init(struct device_node *node, struct device_node *parent) +{ + struct fiq_irq_chip *ic; + unsigned int fiq_other; + int base_ipi = 0; + bool use_for_ipi = of_property_read_bool(node, "use-for-ipi"); - aic_irqc = irqc; - irqc->base = regs; + ic = kzalloc(sizeof(*ic), GFP_KERNEL); + if (!ic) + return -ENOMEM; - info = aic_ic_read(irqc, AIC_INFO); - irqc->nr_hw = FIELD_GET(AIC_INFO_NR_HW, info); + fiq_irqc = ic; - irqc->hw_domain = irq_domain_create_linear(of_node_to_fwnode(node), - irqc->nr_hw + AIC_NR_FIQ, - &aic_irq_domain_ops, irqc); - if (WARN_ON(!irqc->hw_domain)) { - iounmap(irqc->base); - kfree(irqc); + ic->domain = irq_domain_create_linear(of_node_to_fwnode(node), + NR_FIQ, &irq_domain_ops, ic); + if (WARN_ON(!ic->domain)) { + kfree(ic); return -ENODEV; } - irq_domain_update_bus_token(irqc->hw_domain, DOMAIN_BUS_WIRED); - - if (aic_init_smp(irqc, node)) { - irq_domain_remove(irqc->hw_domain); - iounmap(irqc->base); - kfree(irqc); - return -ENODEV; + ic->ipi_domain = + irq_domain_create_hierarchy(NULL, + IRQ_DOMAIN_FLAG_IPI_SINGLE, + FIQ_NR_IPI, + __irq_domain_alloc_fwnode(IRQCHIP_FWNODE_NAMED, 0, "fiq-ipi", NULL), + &ipi_domain_ops, ic); + if (ic->ipi_domain) { + if (use_for_ipi) { + irq_domain_update_bus_token(ic->ipi_domain, DOMAIN_BUS_IPI); + + base_ipi =__irq_domain_alloc_irqs(ic->ipi_domain, -1, FIQ_NR_IPI, + NUMA_NO_NODE, NULL, false, NULL); + + printk("base IPI %d\n", base_ipi); + if (base_ipi >= 0) + set_smp_ipi_range(base_ipi, FIQ_NR_IPI); + } } + set_handle_fiq(handle_fiq); if (!is_kernel_in_hyp_mode()) pr_info("Kernel running in EL1, mapping interrupts"); - cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_AIC_STARTING, - "irqchip/apple-aic/ipi:starting", - aic_init_cpu, NULL); + cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_FIQ_STARTING, + "irqchip/apple-fiq/fiq:starting", + fiq_init_cpu, NULL); - vgic_set_kvm_info(&vgic_info); + if (__irq_resolve_mapping(ic->domain, FIQ_OTHER, &fiq_other)) + WARN_ON(request_irq(fiq_other, fiq_handler, IRQF_SHARED, + "PMC FIQ handler", ic) < 0); - pr_info("Initialized with %d IRQs, %d FIQs, %d vIPIs\n", - irqc->nr_hw, AIC_NR_FIQ, AIC_NR_SWIPI); + pr_info("Initialized with %d FIQs, %sused for IPI\n", NR_FIQ, + use_for_ipi ? "" : "not ");; return 0; } diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index f39b34b1387109..0477ff4603ec50 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -100,6 +100,7 @@ enum cpuhp_state { CPUHP_AP_CPU_PM_STARTING, CPUHP_AP_IRQ_GIC_STARTING, CPUHP_AP_IRQ_HIP04_STARTING, + CPUHP_AP_IRQ_APPLE_FIQ_STARTING, CPUHP_AP_IRQ_APPLE_AIC_STARTING, CPUHP_AP_IRQ_ARMADA_XP_STARTING, CPUHP_AP_IRQ_BCM2836_STARTING, From d87a0dc99e28480d67b993f52d066bd2bf58207c Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 19:11:53 +0000 Subject: [PATCH 16/16] apple/t8103.dts: DT changes for FIQ support We need a new node for the FIQ interrupt controller, and either that new node or the existing aic node need to be tagged with "use-for-ipi". Signed-off-by: Pip Cet --- arch/arm64/boot/dts/apple/t8103-j274.dts | 10 ++++++++++ arch/arm64/boot/dts/apple/t8103.dtsi | 8 +++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8103-j274.dts b/arch/arm64/boot/dts/apple/t8103-j274.dts index e0f6775b987834..efcc4e95fe7d1f 100644 --- a/arch/arm64/boot/dts/apple/t8103-j274.dts +++ b/arch/arm64/boot/dts/apple/t8103-j274.dts @@ -34,6 +34,16 @@ }; }; + soc { + aic: interrupt-controller@23b100000 { + /* use-for-ipi; */ + }; + }; + + fiq: interrupt-controller { + use-for-ipi; + }; + memory@800000000 { device_type = "memory"; reg = <0x8 0 0x2 0>; /* To be filled by loader */ diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index f0c10d1c6cf2ff..d290fde187f826 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -85,9 +85,15 @@ }; }; + fiq: interrupt-controller { + compatible = "apple,fiq"; + #interrupt-cells = <3>; + interrupt-controller; + }; + timer { compatible = "arm,armv8-timer"; - interrupt-parent = <&aic>; + interrupt-parent = <&fiq>; interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; interrupts = , ,