| // SPDX-License-Identifier: GPL-2.0-only |
| /* |
| * Copyright (C) 2012-2015 - ARM Ltd |
| * Author: Marc Zyngier <marc.zyngier@arm.com> |
| */ |
| |
| #include <hyp/adjust_pc.h> |
| |
| #include <linux/compiler.h> |
| #include <linux/irqchip/arm-gic-v3.h> |
| #include <linux/kvm_host.h> |
| |
| #include <asm/kvm_emulate.h> |
| #include <asm/kvm_hyp.h> |
| #include <asm/kvm_mmu.h> |
| |
| #define vtr_to_max_lr_idx(v) ((v) & 0xf) |
| #define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1) |
| #define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5)) |
| |
| static u64 __gic_v3_get_lr(unsigned int lr) |
| { |
| switch (lr & 0xf) { |
| case 0: |
| return read_gicreg(ICH_LR0_EL2); |
| case 1: |
| return read_gicreg(ICH_LR1_EL2); |
| case 2: |
| return read_gicreg(ICH_LR2_EL2); |
| case 3: |
| return read_gicreg(ICH_LR3_EL2); |
| case 4: |
| return read_gicreg(ICH_LR4_EL2); |
| case 5: |
| return read_gicreg(ICH_LR5_EL2); |
| case 6: |
| return read_gicreg(ICH_LR6_EL2); |
| case 7: |
| return read_gicreg(ICH_LR7_EL2); |
| case 8: |
| return read_gicreg(ICH_LR8_EL2); |
| case 9: |
| return read_gicreg(ICH_LR9_EL2); |
| case 10: |
| return read_gicreg(ICH_LR10_EL2); |
| case 11: |
| return read_gicreg(ICH_LR11_EL2); |
| case 12: |
| return read_gicreg(ICH_LR12_EL2); |
| case 13: |
| return read_gicreg(ICH_LR13_EL2); |
| case 14: |
| return read_gicreg(ICH_LR14_EL2); |
| case 15: |
| return read_gicreg(ICH_LR15_EL2); |
| } |
| |
| unreachable(); |
| } |
| |
| static void __gic_v3_set_lr(u64 val, int lr) |
| { |
| switch (lr & 0xf) { |
| case 0: |
| write_gicreg(val, ICH_LR0_EL2); |
| break; |
| case 1: |
| write_gicreg(val, ICH_LR1_EL2); |
| break; |
| case 2: |
| write_gicreg(val, ICH_LR2_EL2); |
| break; |
| case 3: |
| write_gicreg(val, ICH_LR3_EL2); |
| break; |
| case 4: |
| write_gicreg(val, ICH_LR4_EL2); |
| break; |
| case 5: |
| write_gicreg(val, ICH_LR5_EL2); |
| break; |
| case 6: |
| write_gicreg(val, ICH_LR6_EL2); |
| break; |
| case 7: |
| write_gicreg(val, ICH_LR7_EL2); |
| break; |
| case 8: |
| write_gicreg(val, ICH_LR8_EL2); |
| break; |
| case 9: |
| write_gicreg(val, ICH_LR9_EL2); |
| break; |
| case 10: |
| write_gicreg(val, ICH_LR10_EL2); |
| break; |
| case 11: |
| write_gicreg(val, ICH_LR11_EL2); |
| break; |
| case 12: |
| write_gicreg(val, ICH_LR12_EL2); |
| break; |
| case 13: |
| write_gicreg(val, ICH_LR13_EL2); |
| break; |
| case 14: |
| write_gicreg(val, ICH_LR14_EL2); |
| break; |
| case 15: |
| write_gicreg(val, ICH_LR15_EL2); |
| break; |
| } |
| } |
| |
| static void __vgic_v3_write_ap0rn(u32 val, int n) |
| { |
| switch (n) { |
| case 0: |
| write_gicreg(val, ICH_AP0R0_EL2); |
| break; |
| case 1: |
| write_gicreg(val, ICH_AP0R1_EL2); |
| break; |
| case 2: |
| write_gicreg(val, ICH_AP0R2_EL2); |
| break; |
| case 3: |
| write_gicreg(val, ICH_AP0R3_EL2); |
| break; |
| } |
| } |
| |
| static void __vgic_v3_write_ap1rn(u32 val, int n) |
| { |
| switch (n) { |
| case 0: |
| write_gicreg(val, ICH_AP1R0_EL2); |
| break; |
| case 1: |
| write_gicreg(val, ICH_AP1R1_EL2); |
| break; |
| case 2: |
| write_gicreg(val, ICH_AP1R2_EL2); |
| break; |
| case 3: |
| write_gicreg(val, ICH_AP1R3_EL2); |
| break; |
| } |
| } |
| |
| static u32 __vgic_v3_read_ap0rn(int n) |
| { |
| u32 val; |
| |
| switch (n) { |
| case 0: |
| val = read_gicreg(ICH_AP0R0_EL2); |
| break; |
| case 1: |
| val = read_gicreg(ICH_AP0R1_EL2); |
| break; |
| case 2: |
| val = read_gicreg(ICH_AP0R2_EL2); |
| break; |
| case 3: |
| val = read_gicreg(ICH_AP0R3_EL2); |
| break; |
| default: |
| unreachable(); |
| } |
| |
| return val; |
| } |
| |
| static u32 __vgic_v3_read_ap1rn(int n) |
| { |
| u32 val; |
| |
| switch (n) { |
| case 0: |
| val = read_gicreg(ICH_AP1R0_EL2); |
| break; |
| case 1: |
| val = read_gicreg(ICH_AP1R1_EL2); |
| break; |
| case 2: |
| val = read_gicreg(ICH_AP1R2_EL2); |
| break; |
| case 3: |
| val = read_gicreg(ICH_AP1R3_EL2); |
| break; |
| default: |
| unreachable(); |
| } |
| |
| return val; |
| } |
| |
| void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if) |
| { |
| u64 used_lrs = cpu_if->used_lrs; |
| |
| /* |
| * Make sure stores to the GIC via the memory mapped interface |
| * are now visible to the system register interface when reading the |
| * LRs, and when reading back the VMCR on non-VHE systems. |
| */ |
| if (used_lrs || !has_vhe()) { |
| if (!cpu_if->vgic_sre) { |
| dsb(sy); |
| isb(); |
| } |
| } |
| |
| if (used_lrs || cpu_if->its_vpe.its_vm) { |
| int i; |
| u32 elrsr; |
| |
| elrsr = read_gicreg(ICH_ELRSR_EL2); |
| |
| write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EN, ICH_HCR_EL2); |
| |
| for (i = 0; i < used_lrs; i++) { |
| if (elrsr & (1 << i)) |
| cpu_if->vgic_lr[i] &= ~ICH_LR_STATE; |
| else |
| cpu_if->vgic_lr[i] = __gic_v3_get_lr(i); |
| |
| __gic_v3_set_lr(0, i); |
| } |
| } |
| } |
| |
| void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if) |
| { |
| u64 used_lrs = cpu_if->used_lrs; |
| int i; |
| |
| if (used_lrs || cpu_if->its_vpe.its_vm) { |
| write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2); |
| |
| for (i = 0; i < used_lrs; i++) |
| __gic_v3_set_lr(cpu_if->vgic_lr[i], i); |
| } |
| |
| /* |
| * Ensure that writes to the LRs, and on non-VHE systems ensure that |
| * the write to the VMCR in __vgic_v3_activate_traps(), will have |
| * reached the (re)distributors. This ensure the guest will read the |
| * correct values from the memory-mapped interface. |
| */ |
| if (used_lrs || !has_vhe()) { |
| if (!cpu_if->vgic_sre) { |
| isb(); |
| dsb(sy); |
| } |
| } |
| } |
| |
| void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if) |
| { |
| /* |
| * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a |
| * Group0 interrupt (as generated in GICv2 mode) to be |
| * delivered as a FIQ to the guest, with potentially fatal |
| * consequences. So we must make sure that ICC_SRE_EL1 has |
| * been actually programmed with the value we want before |
| * starting to mess with the rest of the GIC, and VMCR_EL2 in |
| * particular. This logic must be called before |
| * __vgic_v3_restore_state(). |
| */ |
| if (!cpu_if->vgic_sre) { |
| write_gicreg(0, ICC_SRE_EL1); |
| isb(); |
| write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2); |
| |
| |
| if (has_vhe()) { |
| /* |
| * Ensure that the write to the VMCR will have reached |
| * the (re)distributors. This ensure the guest will |
| * read the correct values from the memory-mapped |
| * interface. |
| */ |
| isb(); |
| dsb(sy); |
| } |
| } |
| |
| /* |
| * Prevent the guest from touching the GIC system registers if |
| * SRE isn't enabled for GICv3 emulation. |
| */ |
| write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE, |
| ICC_SRE_EL2); |
| |
| /* |
| * If we need to trap system registers, we must write |
| * ICH_HCR_EL2 anyway, even if no interrupts are being |
| * injected, |
| */ |
| if (static_branch_unlikely(&vgic_v3_cpuif_trap) || |
| cpu_if->its_vpe.its_vm) |
| write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2); |
| } |
| |
| void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if) |
| { |
| u64 val; |
| |
| if (!cpu_if->vgic_sre) { |
| cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2); |
| } |
| |
| val = read_gicreg(ICC_SRE_EL2); |
| write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2); |
| |
| if (!cpu_if->vgic_sre) { |
| /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */ |
| isb(); |
| write_gicreg(1, ICC_SRE_EL1); |
| } |
| |
| /* |
| * If we were trapping system registers, we enabled the VGIC even if |
| * no interrupts were being injected, and we disable it again here. |
| */ |
| if (static_branch_unlikely(&vgic_v3_cpuif_trap) || |
| cpu_if->its_vpe.its_vm) |
| write_gicreg(0, ICH_HCR_EL2); |
| } |
| |
| static void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if) |
| { |
| u64 val; |
| u32 nr_pre_bits; |
| |
| val = read_gicreg(ICH_VTR_EL2); |
| nr_pre_bits = vtr_to_nr_pre_bits(val); |
| |
| switch (nr_pre_bits) { |
| case 7: |
| cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3); |
| cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2); |
| fallthrough; |
| case 6: |
| cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1); |
| fallthrough; |
| default: |
| cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0); |
| } |
| |
| switch (nr_pre_bits) { |
| case 7: |
| cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3); |
| cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2); |
| fallthrough; |
| case 6: |
| cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1); |
| fallthrough; |
| default: |
| cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0); |
| } |
| } |
| |
| static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if) |
| { |
| u64 val; |
| u32 nr_pre_bits; |
| |
| val = read_gicreg(ICH_VTR_EL2); |
| nr_pre_bits = vtr_to_nr_pre_bits(val); |
| |
| switch (nr_pre_bits) { |
| case 7: |
| __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3); |
| __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2); |
| fallthrough; |
| case 6: |
| __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1); |
| fallthrough; |
| default: |
| __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0); |
| } |
| |
| switch (nr_pre_bits) { |
| case 7: |
| __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3); |
| __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2); |
| fallthrough; |
| case 6: |
| __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1); |
| fallthrough; |
| default: |
| __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0); |
| } |
| } |
| |
| void __vgic_v3_init_lrs(void) |
| { |
| int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2)); |
| int i; |
| |
| for (i = 0; i <= max_lr_idx; i++) |
| __gic_v3_set_lr(0, i); |
| } |
| |
| /* |
| * Return the GIC CPU configuration: |
| * - [31:0] ICH_VTR_EL2 |
| * - [62:32] RES0 |
| * - [63] MMIO (GICv2) capable |
| */ |
| u64 __vgic_v3_get_gic_config(void) |
| { |
| u64 val, sre = read_gicreg(ICC_SRE_EL1); |
| unsigned long flags = 0; |
| |
| /* |
| * To check whether we have a MMIO-based (GICv2 compatible) |
| * CPU interface, we need to disable the system register |
| * view. To do that safely, we have to prevent any interrupt |
| * from firing (which would be deadly). |
| * |
| * Note that this only makes sense on VHE, as interrupts are |
| * already masked for nVHE as part of the exception entry to |
| * EL2. |
| */ |
| if (has_vhe()) |
| flags = local_daif_save(); |
| |
| /* |
| * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates |
| * that to be able to set ICC_SRE_EL1.SRE to 0, all the |
| * interrupt overrides must be set. You've got to love this. |
| */ |
| sysreg_clear_set(hcr_el2, 0, HCR_AMO | HCR_FMO | HCR_IMO); |
| isb(); |
| write_gicreg(0, ICC_SRE_EL1); |
| isb(); |
| |
| val = read_gicreg(ICC_SRE_EL1); |
| |
| write_gicreg(sre, ICC_SRE_EL1); |
| isb(); |
| sysreg_clear_set(hcr_el2, HCR_AMO | HCR_FMO | HCR_IMO, 0); |
| isb(); |
| |
| if (has_vhe()) |
| local_daif_restore(flags); |
| |
| val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63); |
| val |= read_gicreg(ICH_VTR_EL2); |
| |
| return val; |
| } |
| |
| static u64 __vgic_v3_read_vmcr(void) |
| { |
| return read_gicreg(ICH_VMCR_EL2); |
| } |
| |
| static void __vgic_v3_write_vmcr(u32 vmcr) |
| { |
| write_gicreg(vmcr, ICH_VMCR_EL2); |
| } |
| |
| void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if) |
| { |
| __vgic_v3_save_aprs(cpu_if); |
| if (cpu_if->vgic_sre) |
| cpu_if->vgic_vmcr = __vgic_v3_read_vmcr(); |
| } |
| |
| void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if) |
| { |
| /* |
| * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen |
| * is dependent on ICC_SRE_EL1.SRE, and we have to perform the |
| * VMCR_EL2 save/restore in the world switch. |
| */ |
| if (cpu_if->vgic_sre) |
| __vgic_v3_write_vmcr(cpu_if->vgic_vmcr); |
| __vgic_v3_restore_aprs(cpu_if); |
| } |
| |
| static int __vgic_v3_bpr_min(void) |
| { |
| /* See Pseudocode for VPriorityGroup */ |
| return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2)); |
| } |
| |
| static int __vgic_v3_get_group(struct kvm_vcpu *vcpu) |
| { |
| u32 esr = kvm_vcpu_get_esr(vcpu); |
| u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT; |
| |
| return crm != 8; |
| } |
| |
| #define GICv3_IDLE_PRIORITY 0xff |
| |
| static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr, |
| u64 *lr_val) |
| { |
| unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs; |
| u8 priority = GICv3_IDLE_PRIORITY; |
| int i, lr = -1; |
| |
| for (i = 0; i < used_lrs; i++) { |
| u64 val = __gic_v3_get_lr(i); |
| u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; |
| |
| /* Not pending in the state? */ |
| if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT) |
| continue; |
| |
| /* Group-0 interrupt, but Group-0 disabled? */ |
| if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK)) |
| continue; |
| |
| /* Group-1 interrupt, but Group-1 disabled? */ |
| if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK)) |
| continue; |
| |
| /* Not the highest priority? */ |
| if (lr_prio >= priority) |
| continue; |
| |
| /* This is a candidate */ |
| priority = lr_prio; |
| *lr_val = val; |
| lr = i; |
| } |
| |
| if (lr == -1) |
| *lr_val = ICC_IAR1_EL1_SPURIOUS; |
| |
| return lr; |
| } |
| |
| static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid, |
| u64 *lr_val) |
| { |
| unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs; |
| int i; |
| |
| for (i = 0; i < used_lrs; i++) { |
| u64 val = __gic_v3_get_lr(i); |
| |
| if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid && |
| (val & ICH_LR_ACTIVE_BIT)) { |
| *lr_val = val; |
| return i; |
| } |
| } |
| |
| *lr_val = ICC_IAR1_EL1_SPURIOUS; |
| return -1; |
| } |
| |
| static int __vgic_v3_get_highest_active_priority(void) |
| { |
| u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); |
| u32 hap = 0; |
| int i; |
| |
| for (i = 0; i < nr_apr_regs; i++) { |
| u32 val; |
| |
| /* |
| * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers |
| * contain the active priority levels for this VCPU |
| * for the maximum number of supported priority |
| * levels, and we return the full priority level only |
| * if the BPR is programmed to its minimum, otherwise |
| * we return a combination of the priority level and |
| * subpriority, as determined by the setting of the |
| * BPR, but without the full subpriority. |
| */ |
| val = __vgic_v3_read_ap0rn(i); |
| val |= __vgic_v3_read_ap1rn(i); |
| if (!val) { |
| hap += 32; |
| continue; |
| } |
| |
| return (hap + __ffs(val)) << __vgic_v3_bpr_min(); |
| } |
| |
| return GICv3_IDLE_PRIORITY; |
| } |
| |
| static unsigned int __vgic_v3_get_bpr0(u32 vmcr) |
| { |
| return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT; |
| } |
| |
| static unsigned int __vgic_v3_get_bpr1(u32 vmcr) |
| { |
| unsigned int bpr; |
| |
| if (vmcr & ICH_VMCR_CBPR_MASK) { |
| bpr = __vgic_v3_get_bpr0(vmcr); |
| if (bpr < 7) |
| bpr++; |
| } else { |
| bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT; |
| } |
| |
| return bpr; |
| } |
| |
| /* |
| * Convert a priority to a preemption level, taking the relevant BPR |
| * into account by zeroing the sub-priority bits. |
| */ |
| static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp) |
| { |
| unsigned int bpr; |
| |
| if (!grp) |
| bpr = __vgic_v3_get_bpr0(vmcr) + 1; |
| else |
| bpr = __vgic_v3_get_bpr1(vmcr); |
| |
| return pri & (GENMASK(7, 0) << bpr); |
| } |
| |
| /* |
| * The priority value is independent of any of the BPR values, so we |
| * normalize it using the minimal BPR value. This guarantees that no |
| * matter what the guest does with its BPR, we can always set/get the |
| * same value of a priority. |
| */ |
| static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp) |
| { |
| u8 pre, ap; |
| u32 val; |
| int apr; |
| |
| pre = __vgic_v3_pri_to_pre(pri, vmcr, grp); |
| ap = pre >> __vgic_v3_bpr_min(); |
| apr = ap / 32; |
| |
| if (!grp) { |
| val = __vgic_v3_read_ap0rn(apr); |
| __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr); |
| } else { |
| val = __vgic_v3_read_ap1rn(apr); |
| __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr); |
| } |
| } |
| |
| static int __vgic_v3_clear_highest_active_priority(void) |
| { |
| u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); |
| u32 hap = 0; |
| int i; |
| |
| for (i = 0; i < nr_apr_regs; i++) { |
| u32 ap0, ap1; |
| int c0, c1; |
| |
| ap0 = __vgic_v3_read_ap0rn(i); |
| ap1 = __vgic_v3_read_ap1rn(i); |
| if (!ap0 && !ap1) { |
| hap += 32; |
| continue; |
| } |
| |
| c0 = ap0 ? __ffs(ap0) : 32; |
| c1 = ap1 ? __ffs(ap1) : 32; |
| |
| /* Always clear the LSB, which is the highest priority */ |
| if (c0 < c1) { |
| ap0 &= ~BIT(c0); |
| __vgic_v3_write_ap0rn(ap0, i); |
| hap += c0; |
| } else { |
| ap1 &= ~BIT(c1); |
| __vgic_v3_write_ap1rn(ap1, i); |
| hap += c1; |
| } |
| |
| /* Rescale to 8 bits of priority */ |
| return hap << __vgic_v3_bpr_min(); |
| } |
| |
| return GICv3_IDLE_PRIORITY; |
| } |
| |
| static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| u64 lr_val; |
| u8 lr_prio, pmr; |
| int lr, grp; |
| |
| grp = __vgic_v3_get_group(vcpu); |
| |
| lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val); |
| if (lr < 0) |
| goto spurious; |
| |
| if (grp != !!(lr_val & ICH_LR_GROUP)) |
| goto spurious; |
| |
| pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT; |
| lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; |
| if (pmr <= lr_prio) |
| goto spurious; |
| |
| if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp)) |
| goto spurious; |
| |
| lr_val &= ~ICH_LR_STATE; |
| lr_val |= ICH_LR_ACTIVE_BIT; |
| __gic_v3_set_lr(lr_val, lr); |
| __vgic_v3_set_active_priority(lr_prio, vmcr, grp); |
| vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK); |
| return; |
| |
| spurious: |
| vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS); |
| } |
| |
| static void __vgic_v3_clear_active_lr(int lr, u64 lr_val) |
| { |
| lr_val &= ~ICH_LR_ACTIVE_BIT; |
| if (lr_val & ICH_LR_HW) { |
| u32 pid; |
| |
| pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT; |
| gic_write_dir(pid); |
| } |
| |
| __gic_v3_set_lr(lr_val, lr); |
| } |
| |
| static void __vgic_v3_bump_eoicount(void) |
| { |
| u32 hcr; |
| |
| hcr = read_gicreg(ICH_HCR_EL2); |
| hcr += 1 << ICH_HCR_EOIcount_SHIFT; |
| write_gicreg(hcr, ICH_HCR_EL2); |
| } |
| |
| static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| u32 vid = vcpu_get_reg(vcpu, rt); |
| u64 lr_val; |
| int lr; |
| |
| /* EOImode == 0, nothing to be done here */ |
| if (!(vmcr & ICH_VMCR_EOIM_MASK)) |
| return; |
| |
| /* No deactivate to be performed on an LPI */ |
| if (vid >= VGIC_MIN_LPI) |
| return; |
| |
| lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val); |
| if (lr == -1) { |
| __vgic_v3_bump_eoicount(); |
| return; |
| } |
| |
| __vgic_v3_clear_active_lr(lr, lr_val); |
| } |
| |
| static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| u32 vid = vcpu_get_reg(vcpu, rt); |
| u64 lr_val; |
| u8 lr_prio, act_prio; |
| int lr, grp; |
| |
| grp = __vgic_v3_get_group(vcpu); |
| |
| /* Drop priority in any case */ |
| act_prio = __vgic_v3_clear_highest_active_priority(); |
| |
| lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val); |
| if (lr == -1) { |
| /* Do not bump EOIcount for LPIs that aren't in the LRs */ |
| if (!(vid >= VGIC_MIN_LPI)) |
| __vgic_v3_bump_eoicount(); |
| return; |
| } |
| |
| /* EOImode == 1 and not an LPI, nothing to be done here */ |
| if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI)) |
| return; |
| |
| lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; |
| |
| /* If priorities or group do not match, the guest has fscked-up. */ |
| if (grp != !!(lr_val & ICH_LR_GROUP) || |
| __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio) |
| return; |
| |
| /* Let's now perform the deactivation */ |
| __vgic_v3_clear_active_lr(lr, lr_val); |
| } |
| |
| static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK)); |
| } |
| |
| static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK)); |
| } |
| |
| static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| u64 val = vcpu_get_reg(vcpu, rt); |
| |
| if (val & 1) |
| vmcr |= ICH_VMCR_ENG0_MASK; |
| else |
| vmcr &= ~ICH_VMCR_ENG0_MASK; |
| |
| __vgic_v3_write_vmcr(vmcr); |
| } |
| |
| static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| u64 val = vcpu_get_reg(vcpu, rt); |
| |
| if (val & 1) |
| vmcr |= ICH_VMCR_ENG1_MASK; |
| else |
| vmcr &= ~ICH_VMCR_ENG1_MASK; |
| |
| __vgic_v3_write_vmcr(vmcr); |
| } |
| |
| static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr)); |
| } |
| |
| static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr)); |
| } |
| |
| static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| u64 val = vcpu_get_reg(vcpu, rt); |
| u8 bpr_min = __vgic_v3_bpr_min() - 1; |
| |
| /* Enforce BPR limiting */ |
| if (val < bpr_min) |
| val = bpr_min; |
| |
| val <<= ICH_VMCR_BPR0_SHIFT; |
| val &= ICH_VMCR_BPR0_MASK; |
| vmcr &= ~ICH_VMCR_BPR0_MASK; |
| vmcr |= val; |
| |
| __vgic_v3_write_vmcr(vmcr); |
| } |
| |
| static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| u64 val = vcpu_get_reg(vcpu, rt); |
| u8 bpr_min = __vgic_v3_bpr_min(); |
| |
| if (vmcr & ICH_VMCR_CBPR_MASK) |
| return; |
| |
| /* Enforce BPR limiting */ |
| if (val < bpr_min) |
| val = bpr_min; |
| |
| val <<= ICH_VMCR_BPR1_SHIFT; |
| val &= ICH_VMCR_BPR1_MASK; |
| vmcr &= ~ICH_VMCR_BPR1_MASK; |
| vmcr |= val; |
| |
| __vgic_v3_write_vmcr(vmcr); |
| } |
| |
| static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n) |
| { |
| u32 val; |
| |
| if (!__vgic_v3_get_group(vcpu)) |
| val = __vgic_v3_read_ap0rn(n); |
| else |
| val = __vgic_v3_read_ap1rn(n); |
| |
| vcpu_set_reg(vcpu, rt, val); |
| } |
| |
| static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n) |
| { |
| u32 val = vcpu_get_reg(vcpu, rt); |
| |
| if (!__vgic_v3_get_group(vcpu)) |
| __vgic_v3_write_ap0rn(val, n); |
| else |
| __vgic_v3_write_ap1rn(val, n); |
| } |
| |
| static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu, |
| u32 vmcr, int rt) |
| { |
| __vgic_v3_read_apxrn(vcpu, rt, 0); |
| } |
| |
| static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu, |
| u32 vmcr, int rt) |
| { |
| __vgic_v3_read_apxrn(vcpu, rt, 1); |
| } |
| |
| static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| __vgic_v3_read_apxrn(vcpu, rt, 2); |
| } |
| |
| static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| __vgic_v3_read_apxrn(vcpu, rt, 3); |
| } |
| |
| static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| __vgic_v3_write_apxrn(vcpu, rt, 0); |
| } |
| |
| static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| __vgic_v3_write_apxrn(vcpu, rt, 1); |
| } |
| |
| static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| __vgic_v3_write_apxrn(vcpu, rt, 2); |
| } |
| |
| static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| __vgic_v3_write_apxrn(vcpu, rt, 3); |
| } |
| |
| static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| u64 lr_val; |
| int lr, lr_grp, grp; |
| |
| grp = __vgic_v3_get_group(vcpu); |
| |
| lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val); |
| if (lr == -1) |
| goto spurious; |
| |
| lr_grp = !!(lr_val & ICH_LR_GROUP); |
| if (lr_grp != grp) |
| lr_val = ICC_IAR1_EL1_SPURIOUS; |
| |
| spurious: |
| vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK); |
| } |
| |
| static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| vmcr &= ICH_VMCR_PMR_MASK; |
| vmcr >>= ICH_VMCR_PMR_SHIFT; |
| vcpu_set_reg(vcpu, rt, vmcr); |
| } |
| |
| static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| u32 val = vcpu_get_reg(vcpu, rt); |
| |
| val <<= ICH_VMCR_PMR_SHIFT; |
| val &= ICH_VMCR_PMR_MASK; |
| vmcr &= ~ICH_VMCR_PMR_MASK; |
| vmcr |= val; |
| |
| write_gicreg(vmcr, ICH_VMCR_EL2); |
| } |
| |
| static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| u32 val = __vgic_v3_get_highest_active_priority(); |
| vcpu_set_reg(vcpu, rt, val); |
| } |
| |
| static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| u32 vtr, val; |
| |
| vtr = read_gicreg(ICH_VTR_EL2); |
| /* PRIbits */ |
| val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT; |
| /* IDbits */ |
| val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT; |
| /* SEIS */ |
| if (kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_SEIS_MASK) |
| val |= BIT(ICC_CTLR_EL1_SEIS_SHIFT); |
| /* A3V */ |
| val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT; |
| /* EOImode */ |
| val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT; |
| /* CBPR */ |
| val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT; |
| |
| vcpu_set_reg(vcpu, rt, val); |
| } |
| |
| static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) |
| { |
| u32 val = vcpu_get_reg(vcpu, rt); |
| |
| if (val & ICC_CTLR_EL1_CBPR_MASK) |
| vmcr |= ICH_VMCR_CBPR_MASK; |
| else |
| vmcr &= ~ICH_VMCR_CBPR_MASK; |
| |
| if (val & ICC_CTLR_EL1_EOImode_MASK) |
| vmcr |= ICH_VMCR_EOIM_MASK; |
| else |
| vmcr &= ~ICH_VMCR_EOIM_MASK; |
| |
| write_gicreg(vmcr, ICH_VMCR_EL2); |
| } |
| |
| int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu) |
| { |
| int rt; |
| u32 esr; |
| u32 vmcr; |
| void (*fn)(struct kvm_vcpu *, u32, int); |
| bool is_read; |
| u32 sysreg; |
| |
| esr = kvm_vcpu_get_esr(vcpu); |
| if (vcpu_mode_is_32bit(vcpu)) { |
| if (!kvm_condition_valid(vcpu)) { |
| __kvm_skip_instr(vcpu); |
| return 1; |
| } |
| |
| sysreg = esr_cp15_to_sysreg(esr); |
| } else { |
| sysreg = esr_sys64_to_sysreg(esr); |
| } |
| |
| is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ; |
| |
| switch (sysreg) { |
| case SYS_ICC_IAR0_EL1: |
| case SYS_ICC_IAR1_EL1: |
| if (unlikely(!is_read)) |
| return 0; |
| fn = __vgic_v3_read_iar; |
| break; |
| case SYS_ICC_EOIR0_EL1: |
| case SYS_ICC_EOIR1_EL1: |
| if (unlikely(is_read)) |
| return 0; |
| fn = __vgic_v3_write_eoir; |
| break; |
| case SYS_ICC_IGRPEN1_EL1: |
| if (is_read) |
| fn = __vgic_v3_read_igrpen1; |
| else |
| fn = __vgic_v3_write_igrpen1; |
| break; |
| case SYS_ICC_BPR1_EL1: |
| if (is_read) |
| fn = __vgic_v3_read_bpr1; |
| else |
| fn = __vgic_v3_write_bpr1; |
| break; |
| case SYS_ICC_AP0Rn_EL1(0): |
| case SYS_ICC_AP1Rn_EL1(0): |
| if (is_read) |
| fn = __vgic_v3_read_apxr0; |
| else |
| fn = __vgic_v3_write_apxr0; |
| break; |
| case SYS_ICC_AP0Rn_EL1(1): |
| case SYS_ICC_AP1Rn_EL1(1): |
| if (is_read) |
| fn = __vgic_v3_read_apxr1; |
| else |
| fn = __vgic_v3_write_apxr1; |
| break; |
| case SYS_ICC_AP0Rn_EL1(2): |
| case SYS_ICC_AP1Rn_EL1(2): |
| if (is_read) |
| fn = __vgic_v3_read_apxr2; |
| else |
| fn = __vgic_v3_write_apxr2; |
| break; |
| case SYS_ICC_AP0Rn_EL1(3): |
| case SYS_ICC_AP1Rn_EL1(3): |
| if (is_read) |
| fn = __vgic_v3_read_apxr3; |
| else |
| fn = __vgic_v3_write_apxr3; |
| break; |
| case SYS_ICC_HPPIR0_EL1: |
| case SYS_ICC_HPPIR1_EL1: |
| if (unlikely(!is_read)) |
| return 0; |
| fn = __vgic_v3_read_hppir; |
| break; |
| case SYS_ICC_IGRPEN0_EL1: |
| if (is_read) |
| fn = __vgic_v3_read_igrpen0; |
| else |
| fn = __vgic_v3_write_igrpen0; |
| break; |
| case SYS_ICC_BPR0_EL1: |
| if (is_read) |
| fn = __vgic_v3_read_bpr0; |
| else |
| fn = __vgic_v3_write_bpr0; |
| break; |
| case SYS_ICC_DIR_EL1: |
| if (unlikely(is_read)) |
| return 0; |
| fn = __vgic_v3_write_dir; |
| break; |
| case SYS_ICC_RPR_EL1: |
| if (unlikely(!is_read)) |
| return 0; |
| fn = __vgic_v3_read_rpr; |
| break; |
| case SYS_ICC_CTLR_EL1: |
| if (is_read) |
| fn = __vgic_v3_read_ctlr; |
| else |
| fn = __vgic_v3_write_ctlr; |
| break; |
| case SYS_ICC_PMR_EL1: |
| if (is_read) |
| fn = __vgic_v3_read_pmr; |
| else |
| fn = __vgic_v3_write_pmr; |
| break; |
| default: |
| return 0; |
| } |
| |
| vmcr = __vgic_v3_read_vmcr(); |
| rt = kvm_vcpu_sys_get_rt(vcpu); |
| fn(vcpu, vmcr, rt); |
| |
| __kvm_skip_instr(vcpu); |
| |
| return 1; |
| } |