diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b5b13a932561fc..9a7b03cbd6ae1e 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -314,6 +314,9 @@ config ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE config SMP def_bool y +config IPI_FUNNELING + def_bool y + config KERNEL_MODE_NEON def_bool y diff --git a/arch/arm64/boot/dts/apple/t8103-j274.dts b/arch/arm64/boot/dts/apple/t8103-j274.dts index e0f6775b987834..efcc4e95fe7d1f 100644 --- a/arch/arm64/boot/dts/apple/t8103-j274.dts +++ b/arch/arm64/boot/dts/apple/t8103-j274.dts @@ -34,6 +34,16 @@ }; }; + soc { + aic: interrupt-controller@23b100000 { + /* use-for-ipi; */ + }; + }; + + fiq: interrupt-controller { + use-for-ipi; + }; + memory@800000000 { device_type = "memory"; reg = <0x8 0 0x2 0>; /* To be filled by loader */ diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index f0c10d1c6cf2ff..d290fde187f826 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -85,9 +85,15 @@ }; }; + fiq: interrupt-controller { + compatible = "apple,fiq"; + #interrupt-cells = <3>; + interrupt-controller; + }; + timer { compatible = "arm,armv8-timer"; - interrupt-parent = <&aic>; + interrupt-parent = <&fiq>; interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; interrupts = , , diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 3f1490bfb938a0..ed58d15d484c2e 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -73,6 +73,7 @@ obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o obj-$(CONFIG_ARM64_MTE) += mte.o obj-y += vdso-wrap.o obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o +obj-$(CONFIG_IPI_FUNNELING) += vipi.o obj-y += probes/ head-y := head.o diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 6f6ff072acbde7..ac5f26281d347c 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -979,10 +979,26 @@ static void ipi_teardown(int cpu) } #endif +#ifdef CONFIG_IPI_FUNNELING +extern int __init vipi_init(struct irq_data *hwirq); +#else +static inline int vipi_init(irqdata) +{ + return -EINVAL; +} +#endif + void __init set_smp_ipi_range(int ipi_base, int n) { int i; + if (n < NR_IPI) { + int ret; + BUG_ON(n < 1); + ret = vipi_init(irq_get_irq_data(ipi_base)); + if (ret >= 0) + return; + } WARN_ON(n < NR_IPI); nr_ipi = min(n, NR_IPI); diff --git a/arch/arm64/kernel/vipi.c b/arch/arm64/kernel/vipi.c new file mode 100644 index 00000000000000..863849cf2e0c51 --- /dev/null +++ b/arch/arm64/kernel/vipi.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright The Asahi Linux Contributors + * + * Based on irq-lpc32xx: + * Copyright 2015-2016 Vladimir Zapolskiy + * Based on irq-bcm2836: + * Copyright 2015 Broadcom + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct vipi_irq_chip { + struct irq_domain *domain; + struct irq_data *hwirq; +}; + +#define NR_SWIPI 32 + +static DEFINE_PER_CPU(atomic_t, vipi_flag); +static DEFINE_PER_CPU(atomic_t, vipi_enable); + +static struct vipi_irq_chip *vipi_irqc; + +static void handle_ipi(struct irq_desc *d); +/* + * IPI irqchip + */ + +static void vipi_mask(struct irq_data *d) +{ + u32 irq_bit = BIT(irqd_to_hwirq(d)); + + /* No specific ordering requirements needed here. */ + atomic_andnot(irq_bit, this_cpu_ptr(&vipi_enable)); +} + +static void vipi_unmask(struct irq_data *d) +{ + struct vipi_irq_chip *ic = irq_data_get_irq_chip_data(d); + u32 irq_bit = BIT(irqd_to_hwirq(d)); + + atomic_or(irq_bit, this_cpu_ptr(&vipi_enable)); + + /* + * The atomic_or() above must complete before the atomic_read() + * below to avoid racing aic_ipi_send_mask(). + */ + smp_mb__after_atomic(); + + /* + * If a pending vIPI was unmasked, raise a HW IPI to ourselves. + * No barriers needed here since this is a self-IPI. + */ + if (atomic_read(this_cpu_ptr(&vipi_flag)) & irq_bit) { + struct cpumask self_mask = { 0, }; + cpumask_set_cpu(smp_processor_id(), &self_mask); + ipi_send_mask(ic->hwirq->irq, &self_mask); + } +} + +static void vipi_send_mask(struct irq_data *d, const struct cpumask *mask) +{ + struct vipi_irq_chip *ic = irq_data_get_irq_chip_data(d); + u32 irq_bit = BIT(irqd_to_hwirq(d)); + int cpu; + bool send = false; + unsigned long pending; + struct cpumask sendmask = *mask; + + for_each_cpu(cpu, mask) { + /* + * This sequence is the mirror of the one in vipi_unmask(); + * see the comment there. Additionally, release semantics + * ensure that the vIPI flag set is ordered after any shared + * memory accesses that precede it. This therefore also pairs + * with the atomic_fetch_andnot in handle_ipi(). + */ + pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&vipi_flag, cpu)); + + /* + * The atomic_fetch_or_release() above must complete before the + * atomic_read() below to avoid racing vipi_unmask(). + */ + smp_mb__after_atomic(); + + if (!(pending & irq_bit) && + (atomic_read(per_cpu_ptr(&vipi_enable, cpu)) & irq_bit)) { + cpumask_set_cpu(cpu, &sendmask); + send = true; + } + } + + /* + * The flag writes must complete before the physical IPI is issued + * to another CPU. This is implied by the control dependency on + * the result of atomic_read_acquire() above, which is itself + * already ordered after the vIPI flag write. + */ + if (send) + ipi_send_mask(ic->hwirq->irq, &sendmask); +} + +static struct irq_chip vipi_chip = { + .name = "VIPI", + .irq_mask = vipi_mask, + .irq_unmask = vipi_unmask, + .ipi_send_mask = vipi_send_mask, +}; + +/* + * IPI IRQ domain + */ + +static void handle_ipi(struct irq_desc *d) +{ + int i; + unsigned long enabled, firing; + + /* + * The mask read does not need to be ordered. Only we can change + * our own mask anyway, so no races are possible here, as long as + * we are properly in the interrupt handler (which is covered by + * the barrier that is part of the top-level AIC handler's readl()). + */ + enabled = atomic_read(this_cpu_ptr(&vipi_enable)); + + /* + * Clear the IPIs we are about to handle. This pairs with the + * atomic_fetch_or_release() in vipi_send_mask(), and needs to be + * ordered after the ic_write() above (to avoid dropping vIPIs) and + * before IPI handling code (to avoid races handling vIPIs before they + * are signaled). The former is taken care of by the release semantics + * of the write portion, while the latter is taken care of by the + * acquire semantics of the read portion. + */ + firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&vipi_flag)) & enabled; + + for_each_set_bit(i, &firing, NR_SWIPI) { + struct irq_desc *nd = + irq_resolve_mapping(vipi_irqc->domain, i); + + handle_irq_desc(nd); + } +} + +static int vipi_alloc(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + irq_set_percpu_devid(virq + i); + irq_domain_set_info(d, virq + i, i, &vipi_chip, d->host_data, + handle_percpu_devid_irq, NULL, NULL); + } + + return 0; +} + +static void vipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) +{ + /* Not freeing IPIs */ + WARN_ON(1); +} + +static const struct irq_domain_ops vipi_domain_ops = { + .alloc = vipi_alloc, + .free = vipi_free, +}; + +static int vipi_init_smp(struct vipi_irq_chip *irqc) +{ + struct irq_domain *vipi_domain; + int base_ipi; + struct fwnode_handle *fwnode; + + fwnode = __irq_domain_alloc_fwnode(IRQCHIP_FWNODE_NAMED, 0, + "vIPI", NULL); + + vipi_domain = irq_domain_create_linear(fwnode, NR_SWIPI, + &vipi_domain_ops, irqc); + if (WARN_ON(!vipi_domain)) + return -ENOMEM; + + vipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; + irq_domain_update_bus_token(vipi_domain, DOMAIN_BUS_IPI); + + base_ipi = __irq_domain_alloc_irqs(vipi_domain, -1, NR_SWIPI, + NUMA_NO_NODE, NULL, false, NULL); + + if (WARN_ON(base_ipi < 0)) { + irq_domain_remove(vipi_domain); + return -ENOMEM; + } + + set_smp_ipi_range(base_ipi, NR_SWIPI); + + irqc->domain = vipi_domain; + + return 0; +} + +int __init vipi_init(struct irq_data *hwirq) +{ + struct vipi_irq_chip *irqc; + + irqc = kzalloc(sizeof(*irqc), GFP_KERNEL); + if (!irqc) + return -ENOMEM; + + irqc->hwirq = hwirq; + + if (vipi_init_smp(irqc)) + return -ENOMEM; + + vipi_irqc = irqc; + + irq_set_handler_locked(hwirq, handle_ipi); + + pr_info("Initialized with %d vIPIs\n", NR_SWIPI); + + return 0; +} diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index f88cbf36a9d28d..3d4e246b7e4547 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -115,4 +115,4 @@ obj-$(CONFIG_SL28CPLD_INTC) += irq-sl28cpld.o obj-$(CONFIG_MACH_REALTEK_RTL) += irq-realtek-rtl.o obj-$(CONFIG_WPCM450_AIC) += irq-wpcm450-aic.o obj-$(CONFIG_IRQ_IDT3243X) += irq-idt3243x.o -obj-$(CONFIG_APPLE_AIC) += irq-apple-aic.o +obj-$(CONFIG_APPLE_AIC) += irq-apple-aic.o irq-apple-fiq.o diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index b8c06bd8659e91..8df10e1fc2ed4e 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -23,24 +23,15 @@ * - Automatic masking on ack * - Default "this CPU" register view and explicit per-CPU views * - * In addition, this driver also handles FIQs, as these are routed to the same - * IRQ vector. These are used for Fast IPIs (TODO), the ARMv8 timer IRQs, and - * performance counters (TODO). - * * Implementation notes: * - * - This driver creates two IRQ domains, one for HW IRQs and internal FIQs, - * and one for IPIs. - * - Since Linux needs more than 2 IPIs, we implement a software IRQ controller - * and funnel all IPIs into one per-CPU IPI (the second "self" IPI is unused). - * - FIQ hwirq numbers are assigned after true hwirqs, and are per-cpu. + * - This driver creates two IRQ domains, one for HW IRQs, and one for + * the single IPI we actually support. + * - Since Linux needs more than 2 IPIs, we rely on the arch IRQ layer + * to funnel IPIs through its own implementation, using just one + * per-CPU real IPI (the second "self" IPI is unused). * - DT bindings use 3-cell form (like GIC): * - <0 nr flags> - hwirq #nr - * - <1 nr flags> - FIQ #nr - * - nr=0 Physical HV timer - * - nr=1 Virtual HV timer - * - nr=2 Physical guest timer - * - nr=3 Virtual guest timer */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -104,91 +95,18 @@ #define MASK_REG(x) (4 * ((x) >> 5)) #define MASK_BIT(x) BIT((x) & GENMASK(4, 0)) -/* - * IMP-DEF sysregs that control FIQ sources - * Note: sysreg-based IPIs are not supported yet. - */ - -/* Core PMC control register */ -#define SYS_IMP_APL_PMCR0_EL1 sys_reg(3, 1, 15, 0, 0) -#define PMCR0_IMODE GENMASK(10, 8) -#define PMCR0_IMODE_OFF 0 -#define PMCR0_IMODE_PMI 1 -#define PMCR0_IMODE_AIC 2 -#define PMCR0_IMODE_HALT 3 -#define PMCR0_IMODE_FIQ 4 -#define PMCR0_IACT BIT(11) - -/* IPI request registers */ -#define SYS_IMP_APL_IPI_RR_LOCAL_EL1 sys_reg(3, 5, 15, 0, 0) -#define SYS_IMP_APL_IPI_RR_GLOBAL_EL1 sys_reg(3, 5, 15, 0, 1) -#define IPI_RR_CPU GENMASK(7, 0) -/* Cluster only used for the GLOBAL register */ -#define IPI_RR_CLUSTER GENMASK(23, 16) -#define IPI_RR_TYPE GENMASK(29, 28) -#define IPI_RR_IMMEDIATE 0 -#define IPI_RR_RETRACT 1 -#define IPI_RR_DEFERRED 2 -#define IPI_RR_NOWAKE 3 - -/* IPI status register */ -#define SYS_IMP_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) -#define IPI_SR_PENDING BIT(0) - -/* Guest timer FIQ enable register */ -#define SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2 sys_reg(3, 5, 15, 1, 3) -#define VM_TMR_FIQ_ENABLE_V BIT(0) -#define VM_TMR_FIQ_ENABLE_P BIT(1) - -/* Deferred IPI countdown register */ -#define SYS_IMP_APL_IPI_CR_EL1 sys_reg(3, 5, 15, 3, 1) - -/* Uncore PMC control register */ -#define SYS_IMP_APL_UPMCR0_EL1 sys_reg(3, 7, 15, 0, 4) -#define UPMCR0_IMODE GENMASK(18, 16) -#define UPMCR0_IMODE_OFF 0 -#define UPMCR0_IMODE_AIC 2 -#define UPMCR0_IMODE_HALT 3 -#define UPMCR0_IMODE_FIQ 4 - -/* Uncore PMC status register */ -#define SYS_IMP_APL_UPMSR_EL1 sys_reg(3, 7, 15, 6, 4) -#define UPMSR_IACT BIT(0) - -#define AIC_NR_FIQ 4 -#define AIC_NR_SWIPI 32 - -/* - * FIQ hwirq index definitions: FIQ sources use the DT binding defines - * directly, except that timers are special. At the irqchip level, the - * two timer types are represented by their access method: _EL0 registers - * or _EL02 registers. In the DT binding, the timers are represented - * by their purpose (HV or guest). This mapping is for when the kernel is - * running at EL2 (with VHE). When the kernel is running at EL1, the - * mapping differs and aic_irq_domain_translate() performs the remapping. - */ - -#define AIC_TMR_EL0_PHYS AIC_TMR_HV_PHYS -#define AIC_TMR_EL0_VIRT AIC_TMR_HV_VIRT -#define AIC_TMR_EL02_PHYS AIC_TMR_GUEST_PHYS -#define AIC_TMR_EL02_VIRT AIC_TMR_GUEST_VIRT - struct aic_irq_chip { void __iomem *base; struct irq_domain *hw_domain; struct irq_domain *ipi_domain; int nr_hw; - int ipi_hwirq; }; -static DEFINE_PER_CPU(uint32_t, aic_fiq_unmasked); - -static DEFINE_PER_CPU(atomic_t, aic_vipi_flag); -static DEFINE_PER_CPU(atomic_t, aic_vipi_enable); +#define AIC_NR_IPI 1 static struct aic_irq_chip *aic_irqc; -static void aic_handle_ipi(struct pt_regs *regs); +static void aic_handle_ipi(int index, struct pt_regs *regs); static u32 aic_ic_read(struct aic_irq_chip *ic, u32 reg) { @@ -216,7 +134,7 @@ static void aic_irq_unmask(struct irq_data *d) { struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - aic_ic_write(ic, AIC_MASK_CLR + MASK_REG(d->hwirq), + aic_ic_write(ic, AIC_MASK_CLR + MASK_REG(irqd_to_hwirq(d)), MASK_BIT(irqd_to_hwirq(d))); } @@ -226,7 +144,7 @@ static void aic_irq_eoi(struct irq_data *d) * Reading the interrupt reason automatically acknowledges and masks * the IRQ, so we just unmask it here if needed. */ - if (!irqd_irq_disabled(d) && !irqd_irq_masked(d)) + if (!irqd_irq_masked(d)) aic_irq_unmask(d); } @@ -245,9 +163,9 @@ static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) irq = FIELD_GET(AIC_EVENT_NUM, event); if (type == AIC_EVENT_TYPE_HW) - handle_domain_irq(aic_irqc->hw_domain, irq, regs); - else if (type == AIC_EVENT_TYPE_IPI && irq == 1) - aic_handle_ipi(regs); + handle_domain_irq(ic->hw_domain, irq, regs); + else if (type == AIC_EVENT_TYPE_IPI) + aic_handle_ipi(0 /* irq */, regs); else if (event != 0) pr_err_ratelimited("Unknown IRQ event %d, %d\n", type, irq); } while (event); @@ -257,6 +175,8 @@ static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) * for them separately. This should never trigger if KVM is working * properly, because it will have already taken care of clearing it * on guest exit before this handler runs. + * + * XXX it would be nice to skip this check. */ if (is_kernel_in_hyp_mode() && (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EN) && read_sysreg_s(SYS_ICH_MISR_EL2) != 0) { @@ -271,14 +191,13 @@ static int aic_irq_set_affinity(struct irq_data *d, irq_hw_number_t hwirq = irqd_to_hwirq(d); struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); int cpu; + u32 mask = 0; - if (force) - cpu = cpumask_first(mask_val); - else - cpu = cpumask_any_and(mask_val, cpu_online_mask); + for_each_cpu(cpu, mask_val) + mask |= BIT(cpu); - aic_ic_write(ic, AIC_TARGET_CPU + hwirq * 4, BIT(cpu)); - irq_data_update_effective_affinity(d, cpumask_of(cpu)); + aic_ic_write(ic, AIC_TARGET_CPU + hwirq * 4, mask); + irq_data_update_effective_affinity(d, mask_val); return IRQ_SET_MASK_OK; } @@ -301,153 +220,6 @@ static struct irq_chip aic_chip = { .irq_set_type = aic_irq_set_type, }; -/* - * FIQ irqchip - */ - -static unsigned long aic_fiq_get_idx(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - - return irqd_to_hwirq(d) - ic->nr_hw; -} - -static void aic_fiq_set_mask(struct irq_data *d) -{ - /* Only the guest timers have real mask bits, unfortunately. */ - switch (aic_fiq_get_idx(d)) { - case AIC_TMR_EL02_PHYS: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_P, 0); - isb(); - break; - case AIC_TMR_EL02_VIRT: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_V, 0); - isb(); - break; - default: - break; - } -} - -static void aic_fiq_clear_mask(struct irq_data *d) -{ - switch (aic_fiq_get_idx(d)) { - case AIC_TMR_EL02_PHYS: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_P); - isb(); - break; - case AIC_TMR_EL02_VIRT: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_V); - isb(); - break; - default: - break; - } -} - -static void aic_fiq_mask(struct irq_data *d) -{ - aic_fiq_set_mask(d); - __this_cpu_and(aic_fiq_unmasked, ~BIT(aic_fiq_get_idx(d))); -} - -static void aic_fiq_unmask(struct irq_data *d) -{ - aic_fiq_clear_mask(d); - __this_cpu_or(aic_fiq_unmasked, BIT(aic_fiq_get_idx(d))); -} - -static void aic_fiq_eoi(struct irq_data *d) -{ - /* We mask to ack (where we can), so we need to unmask at EOI. */ - if (__this_cpu_read(aic_fiq_unmasked) & BIT(aic_fiq_get_idx(d))) - aic_fiq_clear_mask(d); -} - -#define TIMER_FIRING(x) \ - (((x) & (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_MASK | \ - ARCH_TIMER_CTRL_IT_STAT)) == \ - (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_STAT)) - -static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) -{ - /* - * It would be really nice if we had a system register that lets us get - * the FIQ source state without having to peek down into sources... - * but such a register does not seem to exist. - * - * So, we have these potential sources to test for: - * - Fast IPIs (not yet used) - * - The 4 timers (CNTP, CNTV for each of HV and guest) - * - Per-core PMCs (not yet supported) - * - Per-cluster uncore PMCs (not yet supported) - * - * Since not dealing with any of these results in a FIQ storm, - * we check for everything here, even things we don't support yet. - */ - - if (read_sysreg_s(SYS_IMP_APL_IPI_SR_EL1) & IPI_SR_PENDING) { - pr_err_ratelimited("Fast IPI fired. Acking.\n"); - write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); - } - - if (TIMER_FIRING(read_sysreg(cntp_ctl_el0))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL0_PHYS, regs); - - if (TIMER_FIRING(read_sysreg(cntv_ctl_el0))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL0_VIRT, regs); - - if (is_kernel_in_hyp_mode()) { - uint64_t enabled = read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2); - - if ((enabled & VM_TMR_FIQ_ENABLE_P) && - TIMER_FIRING(read_sysreg_s(SYS_CNTP_CTL_EL02))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL02_PHYS, regs); - - if ((enabled & VM_TMR_FIQ_ENABLE_V) && - TIMER_FIRING(read_sysreg_s(SYS_CNTV_CTL_EL02))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL02_VIRT, regs); - } - - if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == - (FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_FIQ) | PMCR0_IACT)) { - /* - * Not supported yet, let's figure out how to handle this when - * we implement these proprietary performance counters. For now, - * just mask it and move on. - */ - pr_err_ratelimited("PMC FIQ fired. Masking.\n"); - sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, - FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); - } - - if (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ && - (read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) { - /* Same story with uncore PMCs */ - pr_err_ratelimited("Uncore PMC FIQ fired. Masking.\n"); - sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, - FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); - } -} - -static int aic_fiq_set_type(struct irq_data *d, unsigned int type) -{ - return (type == IRQ_TYPE_LEVEL_HIGH) ? 0 : -EINVAL; -} - -static struct irq_chip fiq_chip = { - .name = "AIC-FIQ", - .irq_mask = aic_fiq_mask, - .irq_unmask = aic_fiq_unmask, - .irq_ack = aic_fiq_set_mask, - .irq_eoi = aic_fiq_eoi, - .irq_set_type = aic_fiq_set_type, -}; - /* * Main IRQ domain */ @@ -455,17 +227,9 @@ static struct irq_chip fiq_chip = { static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, irq_hw_number_t hw) { - struct aic_irq_chip *ic = id->host_data; - - if (hw < ic->nr_hw) { - irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, - handle_fasteoi_irq, NULL, NULL); - irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); - } else { - irq_set_percpu_devid(irq); - irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, - handle_percpu_devid_irq, NULL, NULL); - } + irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, + handle_fasteoi_irq, NULL, NULL); + irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); return 0; } @@ -486,31 +250,6 @@ static int aic_irq_domain_translate(struct irq_domain *id, return -EINVAL; *hwirq = fwspec->param[1]; break; - case AIC_FIQ: - if (fwspec->param[1] >= AIC_NR_FIQ) - return -EINVAL; - *hwirq = ic->nr_hw + fwspec->param[1]; - - /* - * In EL1 the non-redirected registers are the guest's, - * not EL2's, so remap the hwirqs to match. - */ - if (!is_kernel_in_hyp_mode()) { - switch (fwspec->param[1]) { - case AIC_TMR_GUEST_PHYS: - *hwirq = ic->nr_hw + AIC_TMR_EL0_PHYS; - break; - case AIC_TMR_GUEST_VIRT: - *hwirq = ic->nr_hw + AIC_TMR_EL0_VIRT; - break; - case AIC_TMR_HV_PHYS: - case AIC_TMR_HV_VIRT: - return -ENOENT; - default: - break; - } - } - break; default: return -EINVAL; } @@ -564,70 +303,30 @@ static const struct irq_domain_ops aic_irq_domain_ops = { * IPI irqchip */ -static void aic_ipi_mask(struct irq_data *d) +static int aic_ipi_number(struct irq_data *d) { - u32 irq_bit = BIT(irqd_to_hwirq(d)); + return irqd_to_hwirq(d) ? AIC_IPI_OTHER : AIC_IPI_OTHER; +} - /* No specific ordering requirements needed here. */ - atomic_andnot(irq_bit, this_cpu_ptr(&aic_vipi_enable)); +static void aic_ipi_mask(struct irq_data *d) +{ + aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, aic_ipi_number(d)); } static void aic_ipi_unmask(struct irq_data *d) { - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - u32 irq_bit = BIT(irqd_to_hwirq(d)); - - atomic_or(irq_bit, this_cpu_ptr(&aic_vipi_enable)); - - /* - * The atomic_or() above must complete before the atomic_read() - * below to avoid racing aic_ipi_send_mask(). - */ - smp_mb__after_atomic(); - - /* - * If a pending vIPI was unmasked, raise a HW IPI to ourselves. - * No barriers needed here since this is a self-IPI. - */ - if (atomic_read(this_cpu_ptr(&aic_vipi_flag)) & irq_bit) - aic_ic_write(ic, AIC_IPI_SEND, AIC_IPI_SEND_CPU(smp_processor_id())); + aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, aic_ipi_number(d)); } static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) { struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - u32 irq_bit = BIT(irqd_to_hwirq(d)); u32 send = 0; int cpu; - unsigned long pending; - - for_each_cpu(cpu, mask) { - /* - * This sequence is the mirror of the one in aic_ipi_unmask(); - * see the comment there. Additionally, release semantics - * ensure that the vIPI flag set is ordered after any shared - * memory accesses that precede it. This therefore also pairs - * with the atomic_fetch_andnot in aic_handle_ipi(). - */ - pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&aic_vipi_flag, cpu)); - /* - * The atomic_fetch_or_release() above must complete before the - * atomic_read() below to avoid racing aic_ipi_unmask(). - */ - smp_mb__after_atomic(); + for_each_cpu(cpu, mask) + send |= AIC_IPI_SEND_CPU(cpu); - if (!(pending & irq_bit) && - (atomic_read(per_cpu_ptr(&aic_vipi_enable, cpu)) & irq_bit)) - send |= AIC_IPI_SEND_CPU(cpu); - } - - /* - * The flag writes must complete before the physical IPI is issued - * to another CPU. This is implied by the control dependency on - * the result of atomic_read_acquire() above, which is itself - * already ordered after the vIPI flag write. - */ if (send) aic_ic_write(ic, AIC_IPI_SEND, send); } @@ -643,44 +342,24 @@ static struct irq_chip ipi_chip = { * IPI IRQ domain */ -static void aic_handle_ipi(struct pt_regs *regs) +static void aic_handle_ipi(int index, struct pt_regs *regs) { - int i; - unsigned long enabled, firing; - + struct irq_domain *domain = aic_irqc->ipi_domain; + struct aic_irq_chip *ic = aic_irqc; /* * Ack the IPI. We need to order this after the AIC event read, but * that is enforced by normal MMIO ordering guarantees. */ - aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_OTHER); + aic_ic_write(ic, AIC_IPI_ACK, + aic_ipi_number(irq_domain_get_irq_data(domain, index))); - /* - * The mask read does not need to be ordered. Only we can change - * our own mask anyway, so no races are possible here, as long as - * we are properly in the interrupt handler (which is covered by - * the barrier that is part of the top-level AIC handler's readl()). - */ - enabled = atomic_read(this_cpu_ptr(&aic_vipi_enable)); - - /* - * Clear the IPIs we are about to handle. This pairs with the - * atomic_fetch_or_release() in aic_ipi_send_mask(), and needs to be - * ordered after the aic_ic_write() above (to avoid dropping vIPIs) and - * before IPI handling code (to avoid races handling vIPIs before they - * are signaled). The former is taken care of by the release semantics - * of the write portion, while the latter is taken care of by the - * acquire semantics of the read portion. - */ - firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&aic_vipi_flag)) & enabled; - - for_each_set_bit(i, &firing, AIC_NR_SWIPI) - handle_domain_irq(aic_irqc->ipi_domain, i, regs); + handle_domain_irq(domain, index, regs); /* * No ordering needed here; at worst this just changes the timing of * when the next IPI will be delivered. */ - aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); + aic_ic_write(ic, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); } static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, @@ -712,7 +391,7 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) struct irq_domain *ipi_domain; int base_ipi; - ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_SWIPI, + ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_IPI, &aic_ipi_domain_ops, irqc); if (WARN_ON(!ipi_domain)) return -ENODEV; @@ -720,15 +399,15 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); - base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI, + base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_IPI, NUMA_NO_NODE, NULL, false, NULL); - if (WARN_ON(!base_ipi)) { + if (WARN_ON(base_ipi < 0)) { irq_domain_remove(ipi_domain); return -ENODEV; } - set_smp_ipi_range(base_ipi, AIC_NR_SWIPI); + set_smp_ipi_range(base_ipi, AIC_NR_IPI); irqc->ipi_domain = ipi_domain; @@ -737,34 +416,15 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) static int aic_init_cpu(unsigned int cpu) { - /* Mask all hard-wired per-CPU IRQ/FIQ sources */ - - /* Pending Fast IPI FIQs */ - write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); - - /* Timer FIQs */ - sysreg_clear_set(cntp_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); - sysreg_clear_set(cntv_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); + /* Mask hard-wired per-CPU IRQ sources */ /* EL2-only (VHE mode) IRQ sources */ if (is_kernel_in_hyp_mode()) { - /* Guest timers */ - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, - VM_TMR_FIQ_ENABLE_V | VM_TMR_FIQ_ENABLE_P, 0); - /* vGIC maintenance IRQ */ sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); } - /* PMC FIQ */ - sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, - FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); - - /* Uncore PMC FIQ */ - sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, - FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); - - /* Commit all of the above */ + /* Commit the above */ isb(); /* @@ -782,9 +442,6 @@ static int aic_init_cpu(unsigned int cpu) aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_SELF); aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); - /* Initialize the local mask state */ - __this_cpu_write(aic_fiq_unmasked, 0); - return 0; } @@ -800,6 +457,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p void __iomem *regs; u32 info; struct aic_irq_chip *irqc; + bool use_for_ipi = of_property_read_bool(node, "use-for-ipi"); regs = of_iomap(node, 0); if (WARN_ON(!regs)) @@ -816,7 +474,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p irqc->nr_hw = FIELD_GET(AIC_INFO_NR_HW, info); irqc->hw_domain = irq_domain_create_linear(of_node_to_fwnode(node), - irqc->nr_hw + AIC_NR_FIQ, + irqc->nr_hw, &aic_irq_domain_ops, irqc); if (WARN_ON(!irqc->hw_domain)) { iounmap(irqc->base); @@ -826,7 +484,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p irq_domain_update_bus_token(irqc->hw_domain, DOMAIN_BUS_WIRED); - if (aic_init_smp(irqc, node)) { + if (use_for_ipi && aic_init_smp(irqc, node)) { irq_domain_remove(irqc->hw_domain); iounmap(irqc->base); kfree(irqc); @@ -834,7 +492,6 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p } set_handle_irq(aic_handle_irq); - set_handle_fiq(aic_handle_fiq); for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) aic_ic_write(irqc, AIC_MASK_SET + i * 4, U32_MAX); @@ -843,17 +500,14 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p for (i = 0; i < irqc->nr_hw; i++) aic_ic_write(irqc, AIC_TARGET_CPU + i * 4, 1); - if (!is_kernel_in_hyp_mode()) - pr_info("Kernel running in EL1, mapping interrupts"); - cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_AIC_STARTING, "irqchip/apple-aic/ipi:starting", aic_init_cpu, NULL); vgic_set_kvm_info(&vgic_info); - pr_info("Initialized with %d IRQs, %d FIQs, %d vIPIs\n", - irqc->nr_hw, AIC_NR_FIQ, AIC_NR_SWIPI); + pr_info("Initialized with %d IRQs, 1 IPI, %sused for IPI\n", irqc->nr_hw, + use_for_ipi ? "" : "not "); return 0; } diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c new file mode 100644 index 00000000000000..abd1d9924d9a11 --- /dev/null +++ b/drivers/irqchip/irq-apple-fiq.c @@ -0,0 +1,547 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright The Asahi Linux Contributors + * + * Based on irq-lpc32xx: + * Copyright 2015-2016 Vladimir Zapolskiy + * Based on irq-bcm2836: + * Copyright 2015 Broadcom + */ + +/* + * This driver handles FIQs. These are used for Fast IPIs, the ARMv8 + * timer IRQs, and performance counters (TODO). + * + * Implementation notes: + * + * - This driver creates one IRQ domain for FIQs and another for the + * single IPI that is used. + * - DT bindings use 3-cell form (like GIC): + * - <1 nr flags> - FIQ #nr + * - nr=0 Physical HV timer + * - nr=1 Virtual HV timer + * - nr=2 Physical guest timer + * - nr=3 Virtual guest timer + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* + * IMP-DEF sysregs that control FIQ sources + * Note: sysreg-based IPIs are not supported yet. + */ + +/* Core PMC control register */ +#define SYS_IMP_APL_PMCR0_EL1 sys_reg(3, 1, 15, 0, 0) +#define PMCR0_IMODE GENMASK(10, 8) +#define PMCR0_IMODE_OFF 0 +#define PMCR0_IMODE_PMI 1 +#define PMCR0_IMODE_AIC 2 +#define PMCR0_IMODE_HALT 3 +#define PMCR0_IMODE_FIQ 4 +#define PMCR0_IACT BIT(11) + +/* IPI request registers */ +#define SYS_IMP_APL_IPI_RR_LOCAL_EL1 sys_reg(3, 5, 15, 0, 0) +#define SYS_IMP_APL_IPI_RR_GLOBAL_EL1 sys_reg(3, 5, 15, 0, 1) +#define IPI_RR_CPU GENMASK(7, 0) +/* Cluster only used for the GLOBAL register */ +#define IPI_RR_CLUSTER GENMASK(23, 16) +#define IPI_RR_TYPE GENMASK(29, 28) +#define IPI_RR_IMMEDIATE 0 +#define IPI_RR_RETRACT 1 +#define IPI_RR_DEFERRED 2 +#define IPI_RR_NOWAKE 3 + +/* IPI status register */ +#define SYS_IMP_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) +#define IPI_SR_PENDING BIT(0) + +/* Guest timer FIQ enable register */ +#define SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2 sys_reg(3, 5, 15, 1, 3) +#define VM_TMR_FIQ_ENABLE_V BIT(0) +#define VM_TMR_FIQ_ENABLE_P BIT(1) + +/* Deferred IPI countdown register */ +#define SYS_IMP_APL_IPI_CR_EL1 sys_reg(3, 5, 15, 3, 1) + +/* Uncore PMC control register */ +#define SYS_IMP_APL_UPMCR0_EL1 sys_reg(3, 7, 15, 0, 4) +#define UPMCR0_IMODE GENMASK(18, 16) +#define UPMCR0_IMODE_OFF 0 +#define UPMCR0_IMODE_AIC 2 +#define UPMCR0_IMODE_HALT 3 +#define UPMCR0_IMODE_FIQ 4 + +/* Uncore PMC status register */ +#define SYS_IMP_APL_UPMSR_EL1 sys_reg(3, 7, 15, 6, 4) +#define UPMSR_IACT BIT(0) + +#define NR_FIQ 6 +#define FIQ_NR_IPI 1 + +/* + * FIQ hwirq index definitions: FIQ sources use the DT binding defines + * directly, except that timers are special. At the irqchip level, the + * two timer types are represented by their access method: _EL0 registers + * or _EL02 registers. In the DT binding, the timers are represented + * by their purpose (HV or guest). This mapping is for when the kernel is + * running at EL2 (with VHE). When the kernel is running at EL1, the + * mapping differs and irq_domain_translate() performs the remapping. + */ + +#define FIQ_TMR_HV_PHYS AIC_TMR_HV_PHYS +#define FIQ_TMR_HV_VIRT AIC_TMR_HV_VIRT +#define FIQ_TMR_GUEST_PHYS AIC_TMR_GUEST_PHYS +#define FIQ_TMR_GUEST_VIRT AIC_TMR_GUEST_VIRT + +#define FIQ_TMR_EL0_PHYS FIQ_TMR_HV_PHYS +#define FIQ_TMR_EL0_VIRT FIQ_TMR_HV_VIRT +#define FIQ_TMR_EL02_PHYS FIQ_TMR_GUEST_PHYS +#define FIQ_TMR_EL02_VIRT FIQ_TMR_GUEST_VIRT + + +static DEFINE_PER_CPU(uint32_t, fiq_unmasked); + +/* + * FIQ irqchip + */ + +struct fiq_irq_chip { + struct irq_domain *domain; + struct irq_domain *ipi_domain; +}; + +static struct fiq_irq_chip *fiq_irqc; + +static void fiq_set_mask(struct irq_data *d) +{ + /* Only the guest timers have real mask bits, unfortunately. */ + switch (irqd_to_hwirq(d)) { + case FIQ_TMR_EL02_PHYS: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_P, 0); + isb(); + break; + case FIQ_TMR_EL02_VIRT: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_V, 0); + isb(); + break; + default: + break; + } +} + +static void fiq_clear_mask(struct irq_data *d) +{ + switch (irqd_to_hwirq(d)) { + case FIQ_TMR_EL02_PHYS: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_P); + isb(); + break; + case FIQ_TMR_EL02_VIRT: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_V); + isb(); + break; + default: + break; + } +} + +static void fiq_mask(struct irq_data *d) +{ + fiq_set_mask(d); + __this_cpu_and(fiq_unmasked, ~BIT(irqd_to_hwirq(d))); +} + +static void fiq_unmask(struct irq_data *d) +{ + fiq_clear_mask(d); + __this_cpu_or(fiq_unmasked, BIT(irqd_to_hwirq(d))); +} + +static void fiq_eoi(struct irq_data *d) +{ + /* We mask to ack (where we can), so we need to unmask at EOI. */ + if (__this_cpu_read(fiq_unmasked) & BIT(irqd_to_hwirq(d))) + fiq_clear_mask(d); +} + +static void fiq_ipi_mask(struct irq_data *d) +{ + /* The vIPI layer does not assume IPIs are masked. */ +} + +static void fiq_ipi_unmask(struct irq_data *d) +{ + /* The vIPI layer does not assume IPIs are masked. */ +} + +static void fiq_ipi_eoi(struct irq_data *d) +{ +} + +#define TIMER_FIRING(x) \ + (((x) & (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_MASK | \ + ARCH_TIMER_CTRL_IT_STAT)) == \ + (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_STAT)) + +static void __exception_irq_entry handle_fiq(struct pt_regs *regs) +{ + struct fiq_irq_chip *ic = fiq_irqc; + /* + * It would be really nice if we had a system register that lets us get + * the FIQ source state without having to peek down into sources... + * but such a register does not seem to exist. + * + * So, we have these potential sources to test for: + * - Fast IPIs (not yet used) + * - The 4 timers (CNTP, CNTV for each of HV and guest) + * - Per-core PMCs (not yet supported) + * - Per-cluster uncore PMCs (not yet supported) + * + * Since not dealing with any of these results in a FIQ storm, + * we check for everything here, even things we don't support yet. + */ + + if (read_sysreg_s(SYS_IMP_APL_IPI_SR_EL1) & IPI_SR_PENDING) { + write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); + handle_domain_irq(ic->ipi_domain, 0, regs); + } + + if (TIMER_FIRING(read_sysreg(cntp_ctl_el0))) + + handle_domain_irq(ic->domain, FIQ_TMR_EL0_PHYS, regs); + if (TIMER_FIRING(read_sysreg(cntv_ctl_el0))) + handle_domain_irq(ic->domain, FIQ_TMR_EL0_VIRT, regs); + if (is_kernel_in_hyp_mode()) { + uint64_t enabled = read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2); + + if ((enabled & VM_TMR_FIQ_ENABLE_P) && + TIMER_FIRING(read_sysreg_s(SYS_CNTP_CTL_EL02))) + handle_domain_irq(ic->domain, FIQ_TMR_EL02_PHYS, regs); + + if ((enabled & VM_TMR_FIQ_ENABLE_V) && + TIMER_FIRING(read_sysreg_s(SYS_CNTV_CTL_EL02))) + handle_domain_irq(fiq_irqc->domain, FIQ_TMR_EL02_VIRT, regs); + } + + if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == + (FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_FIQ) | PMCR0_IACT)) { + /* + * Not supported yet, let's figure out how to handle this when + * we implement these proprietary performance counters. For now, + * just mask it and move on. + */ + pr_err_ratelimited("PMC FIQ fired. Masking.\n"); + sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, + FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); + } + + if (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ && + (read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) { + /* Same story with uncore PMCs */ + pr_err_ratelimited("Uncore PMC FIQ fired. Masking.\n"); + sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, + FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); + } +} + +static int fiq_set_type(struct irq_data *d, unsigned int type) +{ + return (type == IRQ_TYPE_LEVEL_HIGH) ? 0 : -EINVAL; +} + +static void fiq_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) +{ + int cpu; + + for_each_cpu(cpu, mask) { + int lcpu = get_cpu(); + + if ((lcpu ^ cpu) & 4) { + u64 val = (FIELD_PREP(IPI_RR_TYPE, IPI_RR_IMMEDIATE) | + FIELD_PREP(IPI_RR_CLUSTER, !!(cpu & 4)) | + (cpu & 3)); + write_sysreg_s(val, SYS_IMP_APL_IPI_RR_GLOBAL_EL1); + } else { + u64 val = cpu & 3; + write_sysreg_s(val, SYS_IMP_APL_IPI_RR_LOCAL_EL1); + } + + put_cpu(); + } + + isb(); +} + +static struct irq_chip fiq_chip = { + .name = "FIQ", + .irq_mask = fiq_mask, + .irq_unmask = fiq_unmask, + .irq_ack = fiq_set_mask, + .irq_eoi = fiq_eoi, + .irq_set_type = fiq_set_type, +}; + +static struct irq_chip fiq_ipi_chip = { + .name = "FIQ-IPI", + .irq_mask = fiq_ipi_mask, + .irq_unmask = fiq_ipi_unmask, + .ipi_send_mask = fiq_ipi_send_mask, +}; + +/* + * Main IRQ domain + */ + +static int irq_domain_map(struct irq_domain *id, unsigned int irq, + irq_hw_number_t hw) +{ + irq_set_percpu_devid(irq); + irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, + handle_percpu_devid_irq, NULL, NULL); + + return 0; +} + +static int irq_domain_translate(struct irq_domain *id, + struct irq_fwspec *fwspec, + unsigned long *hwirq, + unsigned int *type) +{ + if (fwspec->param_count != 3 || !is_of_node(fwspec->fwnode)) + return -EINVAL; + + switch (fwspec->param[0]) { + case AIC_FIQ: + if (fwspec->param[1] >= NR_FIQ) + return -EINVAL; + *hwirq = fwspec->param[1]; + + /* + * In EL1 the non-redirected registers are the guest's, + * not EL2's, so remap the hwirqs to match. + */ + if (!is_kernel_in_hyp_mode()) { + switch (fwspec->param[1]) { + case FIQ_TMR_GUEST_PHYS: + *hwirq = FIQ_TMR_EL0_PHYS; + break; + case FIQ_TMR_GUEST_VIRT: + *hwirq = FIQ_TMR_EL0_VIRT; + break; + case FIQ_TMR_HV_PHYS: + case FIQ_TMR_HV_VIRT: + return -ENOENT; + default: + break; + } + } + break; + default: + return -EINVAL; + } + + *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; + + return 0; +} + +static int irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + unsigned int type = IRQ_TYPE_NONE; + struct irq_fwspec *fwspec = arg; + irq_hw_number_t hwirq; + int i, ret; + + ret = irq_domain_translate(domain, fwspec, &hwirq, &type); + if (ret) + return ret; + + for (i = 0; i < nr_irqs; i++) { + ret = irq_domain_map(domain, virq + i, hwirq + i); + if (ret) + return ret; + } + + return 0; +} +static int ipi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + unsigned int type = IRQ_TYPE_NONE; + irq_hw_number_t hwirq; + int i, ret; + + for (i = 0; i < nr_irqs; i++) { + irq_set_percpu_devid(virq + i); + irq_domain_set_info(domain, virq + i, i, &fiq_ipi_chip, domain->host_data, + handle_percpu_devid_irq, NULL, NULL); + } + + return 0; +} + +static void irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } +} + +static const struct irq_domain_ops irq_domain_ops = { + .translate = irq_domain_translate, + .alloc = irq_domain_alloc, + .free = irq_domain_free, +}; + +static const struct irq_domain_ops ipi_domain_ops = { + .alloc = ipi_domain_alloc, + .free = irq_domain_free, +}; + +static int fiq_init_cpu(unsigned int cpu) +{ + /* Mask all hard-wired per-CPU FIQ sources */ + + /* Pending Fast IPI FIQs */ + write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); + + /* Timer FIQs */ + sysreg_clear_set(cntp_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); + sysreg_clear_set(cntv_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); + + /* EL2-only (VHE mode) IRQ sources */ + if (is_kernel_in_hyp_mode()) { + /* Guest timers */ + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, + VM_TMR_FIQ_ENABLE_V | VM_TMR_FIQ_ENABLE_P, 0); + } + + /* PMC FIQ */ + sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, + FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); + + /* Uncore PMC FIQ */ + sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, + FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); + + /* Commit all of the above */ + isb(); + + /* Initialize the local mask state */ + __this_cpu_write(fiq_unmasked, 0); + + return 0; +} + +/* Regular old IRQ handler for "other" FIQs. This will have to go away + * and forward the PMC FIQs at some point, but for now it's better to + * have the stats that we get from a regular IRQ handler. */ + +static enum irqreturn fiq_handler(int irq, void *ptr) +{ + if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == + (FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_FIQ) | PMCR0_IACT)) { + /* + * Not supported yet, let's figure out how to handle this when + * we implement these proprietary performance counters. For now, + * just mask it and move on. + */ + pr_err_ratelimited("PMC FIQ fired. Masking.\n"); + sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, + FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); + return IRQ_HANDLED; + } else if (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ && + (read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) { + /* Same story with uncore PMCs */ + pr_err_ratelimited("Uncore PMC FIQ fired. Masking.\n"); + sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, + FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static int __init fiq_of_ic_init(struct device_node *node, struct device_node *parent) +{ + struct fiq_irq_chip *ic; + unsigned int fiq_other; + int base_ipi = 0; + bool use_for_ipi = of_property_read_bool(node, "use-for-ipi"); + + ic = kzalloc(sizeof(*ic), GFP_KERNEL); + if (!ic) + return -ENOMEM; + + fiq_irqc = ic; + + ic->domain = irq_domain_create_linear(of_node_to_fwnode(node), + NR_FIQ, &irq_domain_ops, ic); + if (WARN_ON(!ic->domain)) { + kfree(ic); + return -ENODEV; + } + + ic->ipi_domain = + irq_domain_create_hierarchy(NULL, + IRQ_DOMAIN_FLAG_IPI_SINGLE, + FIQ_NR_IPI, + __irq_domain_alloc_fwnode(IRQCHIP_FWNODE_NAMED, 0, "fiq-ipi", NULL), + &ipi_domain_ops, ic); + if (ic->ipi_domain) { + if (use_for_ipi) { + irq_domain_update_bus_token(ic->ipi_domain, DOMAIN_BUS_IPI); + + base_ipi =__irq_domain_alloc_irqs(ic->ipi_domain, -1, FIQ_NR_IPI, + NUMA_NO_NODE, NULL, false, NULL); + + printk("base IPI %d\n", base_ipi); + if (base_ipi >= 0) + set_smp_ipi_range(base_ipi, FIQ_NR_IPI); + } + } + + set_handle_fiq(handle_fiq); + + if (!is_kernel_in_hyp_mode()) + pr_info("Kernel running in EL1, mapping interrupts"); + + cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_FIQ_STARTING, + "irqchip/apple-fiq/fiq:starting", + fiq_init_cpu, NULL); + + if (__irq_resolve_mapping(ic->domain, FIQ_OTHER, &fiq_other)) + WARN_ON(request_irq(fiq_other, fiq_handler, IRQF_SHARED, + "PMC FIQ handler", ic) < 0); + + pr_info("Initialized with %d FIQs, %sused for IPI\n", NR_FIQ, + use_for_ipi ? "" : "not ");; + + return 0; +} + +IRQCHIP_DECLARE(apple_m1_fiq, "apple,fiq", fiq_of_ic_init); diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index f39b34b1387109..0477ff4603ec50 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -100,6 +100,7 @@ enum cpuhp_state { CPUHP_AP_CPU_PM_STARTING, CPUHP_AP_IRQ_GIC_STARTING, CPUHP_AP_IRQ_HIP04_STARTING, + CPUHP_AP_IRQ_APPLE_FIQ_STARTING, CPUHP_AP_IRQ_APPLE_AIC_STARTING, CPUHP_AP_IRQ_ARMADA_XP_STARTING, CPUHP_AP_IRQ_BCM2836_STARTING,