hv: Implement ECV on M2 and later chips

This lets us eliminate HV timer ticks on secondary CPUs, which should
significantly improve the scalability of this whole thing.

(We leave them at 1Hz just in case, since that much won't hurt)

Signed-off-by: Hector Martin <[email protected]>

Author: marcan

src/arm_cpu_regs.h

@@ -25,13 +25,28 @@
 #define CNTHCTL_EL0PCTEN BIT(0)
 
 #define SYS_CNTV_CTL_EL0  sys_reg(3, 3, 14, 3, 1)
+#define SYS_CNTV_CTL_EL02 sys_reg(3, 5, 14, 3, 1)
 #define SYS_CNTP_CTL_EL0  sys_reg(3, 3, 14, 2, 1)
+#define SYS_CNTP_CTL_EL02 sys_reg(3, 5, 14, 2, 1)
 #define SYS_CNTHV_CTL_EL2 sys_reg(3, 4, 14, 3, 1)
 #define SYS_CNTHP_CTL_EL2 sys_reg(3, 4, 14, 2, 1)
 #define CNTx_CTL_ISTATUS  BIT(2)
 #define CNTx_CTL_IMASK    BIT(1)
 #define CNTx_CTL_ENABLE   BIT(0)
 
+#define SYS_CNTV_TVAL_EL0  sys_reg(3, 3, 14, 3, 0)
+#define SYS_CNTV_CTL_EL0   sys_reg(3, 3, 14, 3, 1)
+#define SYS_CNTV_CVAL_EL0  sys_reg(3, 3, 14, 3, 2)
+#define SYS_CNTV_TVAL_EL02 sys_reg(3, 5, 14, 3, 0)
+#define SYS_CNTV_CTL_EL02  sys_reg(3, 5, 14, 3, 1)
+#define SYS_CNTV_CVAL_EL02 sys_reg(3, 5, 14, 3, 2)
+#define SYS_CNTP_TVAL_EL0  sys_reg(3, 3, 14, 2, 0)
+#define SYS_CNTP_CTL_EL0   sys_reg(3, 3, 14, 2, 1)
+#define SYS_CNTP_CVAL_EL0  sys_reg(3, 3, 14, 2, 2)
+#define SYS_CNTP_TVAL_EL02 sys_reg(3, 5, 14, 2, 0)
+#define SYS_CNTP_CTL_EL02  sys_reg(3, 5, 14, 2, 1)
+#define SYS_CNTP_CVAL_EL02 sys_reg(3, 5, 14, 2, 2)
+
 #define SYS_ESR_EL2 sys_reg(3, 4, 5, 2, 0)
 #define ESR_ISS2    GENMASK(36, 32)
 #define ESR_EC      GENMASK(31, 26)

src/hv.c

@@ -12,7 +12,8 @@
 #include "usb.h"
 #include "utils.h"
 
-#define HV_TICK_RATE 1000
+#define HV_TICK_RATE      1000
+#define HV_SLOW_TICK_RATE 1
 
 DECLARE_SPINLOCK(bhl);
 
@@ -22,10 +23,12 @@ void hv_exit_guest(void) __attribute__((noreturn));
 extern char _hv_vectors_start[0];
 
 u64 hv_tick_interval;
+u64 hv_secondary_tick_interval;
 
 int hv_pinned_cpu;
 int hv_want_cpu;
 
+static bool hv_has_ecv;
 static bool hv_should_exit;
 bool hv_started_cpus[MAX_CPUS];
 u32 hv_cpus_in_guest;
@@ -60,9 +63,6 @@ void hv_init(void)
     smp_set_wfe_mode(true);
     hv_wdt_init();
 
-    // Enable physical timer for EL1
-    msr(CNTHCTL_EL2, CNTHCTL_EL1PTEN | CNTHCTL_EL1PCTEN);
-
     hv_pt_init();
 
     // Configure hypervisor defaults
@@ -80,6 +80,21 @@ void hv_init(void)
     // Compute tick interval
     hv_tick_interval = mrs(CNTFRQ_EL0) / HV_TICK_RATE;
 
+    hv_has_ecv = mrs(ID_AA64MMFR0_EL1) & (0xfULL << 60);
+
+    if (hv_has_ecv) {
+        printf("HV: ECV enabled\n");
+        reg_set(CNTHCTL_EL2,
+                CNTHCTL_EL1NVVCT | CNTHCTL_EL1NVPCT | CNTHCTL_EL1TVT | CNTHCTL_EL1PCTEN);
+        hv_secondary_tick_interval = mrs(CNTFRQ_EL0) / HV_SLOW_TICK_RATE;
+    } else {
+        printf("HV: No ECV supported\n");
+        // Enable physical timer for EL1
+        msr(CNTHCTL_EL2, CNTHCTL_EL1PTEN | CNTHCTL_EL1PCTEN);
+
+        hv_secondary_tick_interval = hv_tick_interval;
+    }
+
     // Set deep WFI back to defaults
     reg_mask(SYS_IMP_APL_CYC_OVRD, CYC_OVRD_WFI_MODE_MASK, CYC_OVRD_WFI_MODE(0));
 
@@ -121,7 +136,7 @@ void hv_start(void *entry, u64 regs[4])
     hv_secondary_info.sprr_config = mrs(SYS_IMP_APL_SPRR_CONFIG_EL1);
     hv_secondary_info.gxf_config = mrs(SYS_IMP_APL_GXF_CONFIG_EL1);
 
-    hv_arm_tick();
+    hv_arm_tick(false);
     hv_pinned_cpu = -1;
     hv_want_cpu = -1;
     hv_cpus_in_guest = 1;
@@ -177,7 +192,7 @@ static void hv_init_secondary(struct hv_secondary_info_t *info)
     if (gxf_enabled())
         gl2_call(hv_set_gxf_vbar, 0, 0, 0, 0);
 
-    hv_arm_tick();
+    hv_arm_tick(true);
 }
 
 static void hv_enter_secondary(void *entry, u64 regs[4])
@@ -309,9 +324,12 @@ void hv_set_elr(u64 val)
         return msr(ELR_EL2, val);
 }
 
-void hv_arm_tick(void)
+void hv_arm_tick(bool secondary)
 {
-    msr(CNTP_TVAL_EL0, hv_tick_interval);
+    if (secondary)
+        msr(CNTP_TVAL_EL0, hv_secondary_tick_interval);
+    else
+        msr(CNTP_TVAL_EL0, hv_tick_interval);
     msr(CNTP_CTL_EL0, CNTx_CTL_ENABLE);
 }
 

src/hv.h

@@ -104,7 +104,7 @@ void hv_start_secondary(int cpu, void *entry, u64 regs[4]);
 void hv_rendezvous(void);
 bool hv_switch_cpu(int cpu);
 void hv_pin_cpu(int cpu);
-void hv_arm_tick(void);
+void hv_arm_tick(bool secondary);
 void hv_rearm(void);
 void hv_maybe_exit(void);
 void hv_tick(struct exc_info *ctx);

src/hv_exc.c

@@ -207,6 +207,13 @@ static bool hv_handle_msr(struct exc_info *ctx, u64 iss)
         /* Some kind of timer */
         SYSREG_PASS(sys_reg(3, 7, 15, 1, 1));
         SYSREG_PASS(sys_reg(3, 7, 15, 3, 1));
+        /* Architectural timer, for ECV */
+        SYSREG_MAP(SYS_CNTV_CTL_EL0, SYS_CNTV_CTL_EL02)
+        SYSREG_MAP(SYS_CNTV_CVAL_EL0, SYS_CNTV_CVAL_EL02)
+        SYSREG_MAP(SYS_CNTV_TVAL_EL0, SYS_CNTV_TVAL_EL02)
+        SYSREG_MAP(SYS_CNTP_CTL_EL0, SYS_CNTP_CTL_EL02)
+        SYSREG_MAP(SYS_CNTP_CVAL_EL0, SYS_CNTP_CVAL_EL02)
+        SYSREG_MAP(SYS_CNTP_TVAL_EL0, SYS_CNTP_TVAL_EL02)
         /* Spammy stuff seen on t600x p-cores */
         SYSREG_PASS(sys_reg(3, 2, 15, 12, 0));
         SYSREG_PASS(sys_reg(3, 2, 15, 13, 0));
@@ -451,7 +458,7 @@ void hv_exc_fiq(struct exc_info *ctx)
     if (smp_id() != interruptible_cpu && !(mrs(ISR_EL1) & 0x40) && hv_want_cpu == -1) {
         // Non-interruptible CPU and it was just a timer tick (or spurious), so just update FIQs
         hv_update_fiq();
-        hv_arm_tick();
+        hv_arm_tick(true);
         return;
     }
 
@@ -461,9 +468,12 @@ void hv_exc_fiq(struct exc_info *ctx)
 
     // Only poll for HV events in the interruptible CPU
     if (tick) {
-        if (smp_id() == interruptible_cpu)
+        if (smp_id() == interruptible_cpu) {
             hv_tick(ctx);
-        hv_arm_tick();
+            hv_arm_tick(false);
+        } else {
+            hv_arm_tick(true);
+        }
     }
 
     if (mrs(CNTV_CTL_EL0) == (CNTx_CTL_ISTATUS | CNTx_CTL_ENABLE)) {