Skip to content

Commit

Permalink
hv_exc: Improve multi-core scalability
Browse files Browse the repository at this point in the history
The HV tick polling now only runs on CPU#0. All CPUs have the 1000Hz
HV tick, but secondaries only use it to poll the FIQ state and that path
does not take the BHL if no other FIQ was pending.

Signed-off-by: Hector Martin <[email protected]>
  • Loading branch information
marcan committed Apr 19, 2022
1 parent 3020e26 commit ad9bb7a
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 16 deletions.
14 changes: 11 additions & 3 deletions src/hv.c
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,12 @@ void hv_rearm(void)
msr(CNTP_CTL_EL0, CNTx_CTL_ENABLE);
}

void hv_check_rendezvous(struct exc_info *ctx)
bool hv_want_rendezvous(void)
{
return hv_want_cpu != -1;
}

void hv_do_rendezvous(struct exc_info *ctx)
{
if (hv_want_cpu == smp_id()) {
hv_want_cpu = -1;
Expand All @@ -323,12 +328,15 @@ void hv_check_rendezvous(struct exc_info *ctx)
}
}

void hv_tick(struct exc_info *ctx)
void hv_maybe_exit(void)
{
if (hv_should_exit) {
spin_unlock(&bhl);
hv_exit_guest();
}
}

void hv_tick(struct exc_info *ctx)
{
hv_wdt_pet();
iodev_handle_events(uartproxy_iodev);
if (iodev_can_read(uartproxy_iodev)) {
Expand Down
4 changes: 3 additions & 1 deletion src/hv.h
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,9 @@ void hv_rendezvous(void);
void hv_switch_cpu(int cpu);
void hv_arm_tick(void);
void hv_rearm(void);
void hv_check_rendezvous(struct exc_info *ctx);
bool hv_want_rendezvous(void);
void hv_do_rendezvous(struct exc_info *ctx);
void hv_maybe_exit(void);
void hv_tick(struct exc_info *ctx);

#endif
43 changes: 31 additions & 12 deletions src/hv_exc.c
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,17 @@ extern spinlock_t bhl;
((op2) << ESR_ISS_MSR_OP2_SHIFT))
#define SYSREG_ISS(...) _SYSREG_ISS(__VA_ARGS__)

#define D_PERCPU(t, x) t x[MAX_CPUS]
#define PERCPU(x) x[mrs(TPIDR_EL2)]
#define PERCPU(x) pcpu[mrs(TPIDR_EL2)].x

D_PERCPU(static bool, ipi_queued);
D_PERCPU(static bool, ipi_pending);
D_PERCPU(static bool, pmc_pending);
D_PERCPU(static u64, pmc_irq_mode);
struct hv_pcpu_data {
u32 ipi_queued;
u32 ipi_pending;
u32 pmc_pending;
u64 pmc_irq_mode;
u64 exc_entry_pmcr0_cnt;
} ALIGNED(64);

D_PERCPU(static u64, exc_entry_pmcr0_cnt);
struct hv_pcpu_data pcpu[MAX_CPUS];

void hv_exit_guest(void) __attribute__((noreturn));

Expand Down Expand Up @@ -216,7 +218,7 @@ static bool hv_handle_msr(struct exc_info *ctx, u64 iss)
msr(SYS_IMP_APL_IPI_RR_LOCAL_EL1, regs[rt]);
for (int i = 0; i < MAX_CPUS; i++)
if (mpidr == smp_get_mpidr(i))
ipi_queued[i] = true;
pcpu[i].ipi_queued = true;
return true;
}
case SYSREG_ISS(SYS_IMP_APL_IPI_RR_GLOBAL_EL1):
Expand All @@ -225,7 +227,7 @@ static bool hv_handle_msr(struct exc_info *ctx, u64 iss)
msr(SYS_IMP_APL_IPI_RR_GLOBAL_EL1, regs[rt]);
for (int i = 0; i < MAX_CPUS; i++) {
if (mpidr == (smp_get_mpidr(i) & 0xffff))
ipi_queued[i] = true;
pcpu[i].ipi_queued = true;
}
return true;
case SYSREG_ISS(SYS_IMP_APL_IPI_SR_EL1):
Expand Down Expand Up @@ -366,10 +368,27 @@ void hv_exc_irq(struct exc_info *ctx)

void hv_exc_fiq(struct exc_info *ctx)
{
hv_wdt_breadcrumb('F');
hv_exc_entry(ctx);
bool tick = false;

hv_maybe_exit();

if (mrs(CNTP_CTL_EL0) == (CNTx_CTL_ISTATUS | CNTx_CTL_ENABLE)) {
msr(CNTP_CTL_EL0, CNTx_CTL_ISTATUS | CNTx_CTL_IMASK | CNTx_CTL_ENABLE);
tick = true;
}

if (mrs(TPIDR_EL2) != 0 && !(mrs(ISR_EL1) & 0x40)) {
// Secondary CPU and it was just a timer tick (or spurious), so just update FIQs
hv_update_fiq();
return;
}

// Slow (single threaded) path
hv_wdt_breadcrumb('F');
hv_exc_entry(ctx);

// Only poll for HV events in CPU 0
if (tick && mrs(TPIDR_EL2) == 0) {
hv_tick(ctx);
hv_arm_tick();
}
Expand Down Expand Up @@ -403,7 +422,7 @@ void hv_exc_fiq(struct exc_info *ctx)
msr(SYS_IMP_APL_IPI_SR_EL1, IPI_SR_PENDING);
sysop("isb");
}
hv_check_rendezvous(ctx);
hv_do_rendezvous(ctx);

// Handles guest timers
hv_exc_exit(ctx);
Expand Down

0 comments on commit ad9bb7a

Please sign in to comment.