/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2011 NetApp, Inc.
* All rights reserved.
* Copyright (c) 2019 Joyent, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
#include "opt_bhyve_snapshot.h"
#include <sys/param.h>
#include <sys/lock.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/systm.h>
#include <sys/smp.h>
#include <x86/specialreg.h>
#include <x86/apicreg.h>
#include <machine/clock.h>
#include <machine/smp.h>
#include <machine/vmm.h>
#include <machine/vmm_snapshot.h>
#include "vmm_lapic.h"
#include "vmm_ktr.h"
#include "vmm_stat.h"
#include "vlapic.h"
#include "vlapic_priv.h"
#include "vioapic.h"
#define PRIO(x) ((x) >> 4)
#define VLAPIC_VERSION (0x14)
#define x2apic(vlapic) (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
/*
* The 'vlapic->timer_mtx' is used to provide mutual exclusion between the
* vlapic_callout_handler() and vcpu accesses to:
* - timer_freq_bt, timer_period_bt, timer_fire_bt
* - timer LVT register
*/
#define VLAPIC_TIMER_LOCK(vlapic) mtx_lock_spin(&((vlapic)->timer_mtx))
#define VLAPIC_TIMER_UNLOCK(vlapic) mtx_unlock_spin(&((vlapic)->timer_mtx))
#define VLAPIC_TIMER_LOCKED(vlapic) mtx_owned(&((vlapic)->timer_mtx))
/*
* APIC timer frequency:
* - arbitrary but chosen to be in the ballpark of contemporary hardware.
* - power-of-two to avoid loss of precision when converted to a bintime.
*/
#define VLAPIC_BUS_FREQ (128 * 1024 * 1024)
static void vlapic_set_error(struct vlapic *, uint32_t, bool);
static void vlapic_callout_handler(void *arg);
static void vlapic_reset(struct vlapic *vlapic);
static __inline uint32_t
vlapic_get_id(struct vlapic *vlapic)
{
if (x2apic(vlapic))
return (vlapic->vcpuid);
else
return (vlapic->vcpuid << 24);
}
static uint32_t
x2apic_ldr(struct vlapic *vlapic)
{
int apicid;
uint32_t ldr;
apicid = vlapic_get_id(vlapic);
ldr = 1 << (apicid & 0xf);
ldr |= (apicid & 0xffff0) << 12;
return (ldr);
}
void
vlapic_dfr_write_handler(struct vlapic *vlapic)
{
struct LAPIC *lapic;
lapic = vlapic->apic_page;
if (x2apic(vlapic)) {
VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x",
lapic->dfr);
lapic->dfr = 0;
return;
}
lapic->dfr &= APIC_DFR_MODEL_MASK;
lapic->dfr |= APIC_DFR_RESERVED;
if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model");
else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model");
else
VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr);
}
void
vlapic_ldr_write_handler(struct vlapic *vlapic)
{
struct LAPIC *lapic;
lapic = vlapic->apic_page;
/* LDR is read-only in x2apic mode */
if (x2apic(vlapic)) {
VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x",
lapic->ldr);
lapic->ldr = x2apic_ldr(vlapic);
} else {
lapic->ldr &= ~APIC_LDR_RESERVED;
VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
}
}
void
vlapic_id_write_handler(struct vlapic *vlapic)
{
struct LAPIC *lapic;
/*
* We don't allow the ID register to be modified so reset it back to
* its default value.
*/
lapic = vlapic->apic_page;
lapic->id = vlapic_get_id(vlapic);
}
static int
vlapic_timer_divisor(uint32_t dcr)
{
switch (dcr & 0xB) {
case APIC_TDCR_1:
return (1);
case APIC_TDCR_2:
return (2);
case APIC_TDCR_4:
return (4);
case APIC_TDCR_8:
return (8);
case APIC_TDCR_16:
return (16);
case APIC_TDCR_32:
return (32);
case APIC_TDCR_64:
return (64);
case APIC_TDCR_128:
return (128);
default:
panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
}
}
#if 0
static inline void
vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
{
printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
*lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
*lvt & APIC_LVTT_M);
}
#endif
static uint32_t
vlapic_get_ccr(struct vlapic *vlapic)
{
struct bintime bt_now, bt_rem;
struct LAPIC *lapic __diagused;
uint32_t ccr;
ccr = 0;
lapic = vlapic->apic_page;
VLAPIC_TIMER_LOCK(vlapic);
if (callout_active(&vlapic->callout)) {
/*
* If the timer is scheduled to expire in the future then
* compute the value of 'ccr' based on the remaining time.
*/
binuptime(&bt_now);
if (bintime_cmp(&vlapic->timer_fire_bt, &bt_now, >)) {
bt_rem = vlapic->timer_fire_bt;
bintime_sub(&bt_rem, &bt_now);
ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt);
ccr += bt_rem.frac / vlapic->timer_freq_bt.frac;
}
}
KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, "
"icr_timer is %#x", ccr, lapic->icr_timer));
VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x",
ccr, lapic->icr_timer);
VLAPIC_TIMER_UNLOCK(vlapic);
return (ccr);
}
void
vlapic_dcr_write_handler(struct vlapic *vlapic)
{
struct LAPIC *lapic;
int divisor;
lapic = vlapic->apic_page;
VLAPIC_TIMER_LOCK(vlapic);
divisor = vlapic_timer_divisor(lapic->dcr_timer);
VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d",
lapic->dcr_timer, divisor);
/*
* Update the timer frequency and the timer period.
*
* XXX changes to the frequency divider will not take effect until
* the timer is reloaded.
*/
FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt);
vlapic->timer_period_bt = vlapic->timer_freq_bt;
bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
VLAPIC_TIMER_UNLOCK(vlapic);
}
void
vlapic_esr_write_handler(struct vlapic *vlapic)
{
struct LAPIC *lapic;
lapic = vlapic->apic_page;
lapic->esr = vlapic->esr_pending;
vlapic->esr_pending = 0;
}
int
vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
{
struct LAPIC *lapic;
uint32_t *irrptr, *tmrptr, mask;
int idx;
KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector));
lapic = vlapic->apic_page;
if (!(lapic->svr & APIC_SVR_ENABLE)) {
VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring "
"interrupt %d", vector);
return (0);
}
if (vector < 16) {
vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR,
false);
VLAPIC_CTR1(vlapic, "vlapic ignoring interrupt to vector %d",
vector);
return (1);
}
if (vlapic->ops.set_intr_ready)
return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level));
idx = (vector / 32) * 4;
mask = 1 << (vector % 32);
irrptr = &lapic->irr0;
atomic_set_int(&irrptr[idx], mask);
/*
* Verify that the trigger-mode of the interrupt matches with
* the vlapic TMR registers.
*/
tmrptr = &lapic->tmr0;
if ((tmrptr[idx] & mask) != (level ? mask : 0)) {
VLAPIC_CTR3(vlapic, "vlapic TMR[%d] is 0x%08x but "
"interrupt is %s-triggered", idx / 4, tmrptr[idx],
level ? "level" : "edge");
}
VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
return (1);
}
static __inline uint32_t *
vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset)
{
struct LAPIC *lapic = vlapic->apic_page;
int i;
switch (offset) {
case APIC_OFFSET_CMCI_LVT:
return (&lapic->lvt_cmci);
case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
return ((&lapic->lvt_timer) + i);
default:
panic("vlapic_get_lvt: invalid LVT\n");
}
}
static __inline int
lvt_off_to_idx(uint32_t offset)
{
int index;
switch (offset) {
case APIC_OFFSET_CMCI_LVT:
index = APIC_LVT_CMCI;
break;
case APIC_OFFSET_TIMER_LVT:
index = APIC_LVT_TIMER;
break;
case APIC_OFFSET_THERM_LVT:
index = APIC_LVT_THERMAL;
break;
case APIC_OFFSET_PERF_LVT:
index = APIC_LVT_PMC;
break;
case APIC_OFFSET_LINT0_LVT:
index = APIC_LVT_LINT0;
break;
case APIC_OFFSET_LINT1_LVT:
index = APIC_LVT_LINT1;
break;
case APIC_OFFSET_ERROR_LVT:
index = APIC_LVT_ERROR;
break;
default:
index = -1;
break;
}
KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: "
"invalid lvt index %d for offset %#x", index, offset));
return (index);
}
static __inline uint32_t
vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
{
int idx;
uint32_t val;
idx = lvt_off_to_idx(offset);
val = atomic_load_acq_32(&vlapic->lvt_last[idx]);
return (val);
}
void
vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset)
{
uint32_t *lvtptr, mask, val;
struct LAPIC *lapic;
int idx;
lapic = vlapic->apic_page;
lvtptr = vlapic_get_lvtptr(vlapic, offset);
val = *lvtptr;
idx = lvt_off_to_idx(offset);
if (!(lapic->svr & APIC_SVR_ENABLE))
val |= APIC_LVT_M;
mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR;
switch (offset) {
case APIC_OFFSET_TIMER_LVT:
mask |= APIC_LVTT_TM;
break;
case APIC_OFFSET_ERROR_LVT:
break;
case APIC_OFFSET_LINT0_LVT:
case APIC_OFFSET_LINT1_LVT:
mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP;
/* FALLTHROUGH */
default:
mask |= APIC_LVT_DM;
break;
}
val &= mask;
*lvtptr = val;
atomic_store_rel_32(&vlapic->lvt_last[idx], val);
}
static void
vlapic_mask_lvts(struct vlapic *vlapic)
{
struct LAPIC *lapic = vlapic->apic_page;
lapic->lvt_cmci |= APIC_LVT_M;
vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT);
lapic->lvt_timer |= APIC_LVT_M;
vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT);
lapic->lvt_thermal |= APIC_LVT_M;
vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT);
lapic->lvt_pcint |= APIC_LVT_M;
vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT);
lapic->lvt_lint0 |= APIC_LVT_M;
vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT);
lapic->lvt_lint1 |= APIC_LVT_M;
vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT);
lapic->lvt_error |= APIC_LVT_M;
vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT);
}
static int
vlapic_fire_lvt(struct vlapic *vlapic, u_int lvt)
{
uint32_t mode, reg, vec;
reg = atomic_load_acq_32(&vlapic->lvt_last[lvt]);
if (reg & APIC_LVT_M)
return (0);
vec = reg & APIC_LVT_VECTOR;
mode = reg & APIC_LVT_DM;
switch (mode) {
case APIC_LVT_DM_FIXED:
if (vec < 16) {
vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR,
lvt == APIC_LVT_ERROR);
return (0);
}
if (vlapic_set_intr_ready(vlapic, vec, false))
vcpu_notify_event(vlapic->vcpu, true);
break;
case APIC_LVT_DM_NMI:
vm_inject_nmi(vlapic->vcpu);
break;
case APIC_LVT_DM_EXTINT:
vm_inject_extint(vlapic->vcpu);
break;
default:
// Other modes ignored
return (0);
}
return (1);
}
#if 1
static void
dump_isrvec_stk(struct vlapic *vlapic)
{
int i;
uint32_t *isrptr;
isrptr = &vlapic->apic_page->isr0;
for (i = 0; i < 8; i++)
printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
for (i = 0; i <= vlapic->isrvec_stk_top; i++)
printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
}
#endif
/*
* Algorithm adopted from section "Interrupt, Task and Processor Priority"
* in Intel Architecture Manual Vol 3a.
*/
static void
vlapic_update_ppr(struct vlapic *vlapic)
{
int isrvec, tpr, ppr;
/*
* Note that the value on the stack at index 0 is always 0.
*
* This is a placeholder for the value of ISRV when none of the
* bits is set in the ISRx registers.
*/
isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
tpr = vlapic->apic_page->tpr;
#if 1
{
int i, lastprio, curprio, vector, idx;
uint32_t *isrptr;
if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
panic("isrvec_stk is corrupted: %d", isrvec);
/*
* Make sure that the priority of the nested interrupts is
* always increasing.
*/
lastprio = -1;
for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
curprio = PRIO(vlapic->isrvec_stk[i]);
if (curprio <= lastprio) {
dump_isrvec_stk(vlapic);
panic("isrvec_stk does not satisfy invariant");
}
lastprio = curprio;
}
/*
* Make sure that each bit set in the ISRx registers has a
* corresponding entry on the isrvec stack.
*/
i = 1;
isrptr = &vlapic->apic_page->isr0;
for (vector = 0; vector < 256; vector++) {
idx = (vector / 32) * 4;
if (isrptr[idx] & (1 << (vector % 32))) {
if (i > vlapic->isrvec_stk_top ||
vlapic->isrvec_stk[i] != vector) {
dump_isrvec_stk(vlapic);
panic("ISR and isrvec_stk out of sync");
}
i++;
}
}
}
#endif
if (PRIO(tpr) >= PRIO(isrvec))
ppr = tpr;
else
ppr = isrvec & 0xf0;
vlapic->apic_page->ppr = ppr;
VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
}
void
vlapic_sync_tpr(struct vlapic *vlapic)
{
vlapic_update_ppr(vlapic);
}
static VMM_STAT(VLAPIC_GRATUITOUS_EOI, "EOI without any in-service interrupt");
static void
vlapic_process_eoi(struct vlapic *vlapic)
{
struct LAPIC *lapic = vlapic->apic_page;
uint32_t *isrptr, *tmrptr;
int i, idx, bitpos, vector;
isrptr = &lapic->isr0;
tmrptr = &lapic->tmr0;
for (i = 7; i >= 0; i--) {
idx = i * 4;
bitpos = fls(isrptr[idx]);
if (bitpos-- != 0) {
if (vlapic->isrvec_stk_top <= 0) {
panic("invalid vlapic isrvec_stk_top %d",
vlapic->isrvec_stk_top);
}
isrptr[idx] &= ~(1 << bitpos);
vector = i * 32 + bitpos;
VLAPIC_CTR1(vlapic, "EOI vector %d", vector);
VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
vlapic->isrvec_stk_top--;
vlapic_update_ppr(vlapic);
if ((tmrptr[idx] & (1 << bitpos)) != 0) {
vioapic_process_eoi(vlapic->vm, vector);
}
return;
}
}
VLAPIC_CTR0(vlapic, "Gratuitous EOI");
vmm_stat_incr(vlapic->vcpu, VLAPIC_GRATUITOUS_EOI, 1);
}
static __inline int
vlapic_get_lvt_field(uint32_t lvt, uint32_t mask)
{
return (lvt & mask);
}
static __inline int
vlapic_periodic_timer(struct vlapic *vlapic)
{
uint32_t lvt;
lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
}
static VMM_STAT(VLAPIC_INTR_ERROR, "error interrupts generated by vlapic");
static void
vlapic_set_error(struct vlapic *vlapic, uint32_t mask, bool lvt_error)
{
vlapic->esr_pending |= mask;
/*
* Avoid infinite recursion if the error LVT itself is configured with
* an illegal vector.
*/
if (lvt_error)
return;
if (vlapic_fire_lvt(vlapic, APIC_LVT_ERROR)) {
vmm_stat_incr(vlapic->vcpu, VLAPIC_INTR_ERROR, 1);
}
}
static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
static void
vlapic_fire_timer(struct vlapic *vlapic)
{
KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked"));
if (vlapic_fire_lvt(vlapic, APIC_LVT_TIMER)) {
VLAPIC_CTR0(vlapic, "vlapic timer fired");
vmm_stat_incr(vlapic->vcpu, VLAPIC_INTR_TIMER, 1);
}
}
static VMM_STAT(VLAPIC_INTR_CMC,
"corrected machine check interrupts generated by vlapic");
void
vlapic_fire_cmci(struct vlapic *vlapic)
{
if (vlapic_fire_lvt(vlapic, APIC_LVT_CMCI)) {
vmm_stat_incr(vlapic->vcpu, VLAPIC_INTR_CMC, 1);
}
}
static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1,
"lvts triggered");
int
vlapic_trigger_lvt(struct vlapic *vlapic, int vector)
{
if (vlapic_enabled(vlapic) == false) {
/*
* When the local APIC is global/hardware disabled,
* LINT[1:0] pins are configured as INTR and NMI pins,
* respectively.
*/
switch (vector) {
case APIC_LVT_LINT0:
vm_inject_extint(vlapic->vcpu);
break;
case APIC_LVT_LINT1:
vm_inject_nmi(vlapic->vcpu);
break;
default:
break;
}
return (0);
}
switch (vector) {
case APIC_LVT_LINT0:
case APIC_LVT_LINT1:
case APIC_LVT_TIMER:
case APIC_LVT_ERROR:
case APIC_LVT_PMC:
case APIC_LVT_THERMAL:
case APIC_LVT_CMCI:
if (vlapic_fire_lvt(vlapic, vector)) {
vmm_stat_array_incr(vlapic->vcpu, LVTS_TRIGGERRED,
vector, 1);
}
break;
default:
return (EINVAL);
}
return (0);
}
static void
vlapic_callout_reset(struct vlapic *vlapic, sbintime_t t)
{
callout_reset_sbt_curcpu(&vlapic->callout, t, 0,
vlapic_callout_handler, vlapic, 0);
}
static void
vlapic_callout_handler(void *arg)
{
struct vlapic *vlapic;
struct bintime bt, btnow;
sbintime_t rem_sbt;
vlapic = arg;
VLAPIC_TIMER_LOCK(vlapic);
if (callout_pending(&vlapic->callout)) /* callout was reset */
goto done;
if (!callout_active(&vlapic->callout)) /* callout was stopped */
goto done;
callout_deactivate(&vlapic->callout);
vlapic_fire_timer(vlapic);
if (vlapic_periodic_timer(vlapic)) {
binuptime(&btnow);
KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=),
("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx",
btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec,
vlapic->timer_fire_bt.frac));
/*
* Compute the delta between when the timer was supposed to
* fire and the present time.
*/
bt = btnow;
bintime_sub(&bt, &vlapic->timer_fire_bt);
rem_sbt = bttosbt(vlapic->timer_period_bt);
if (bintime_cmp(&bt, &vlapic->timer_period_bt, <)) {
/*
* Adjust the time until the next countdown downward
* to account for the lost time.
*/
rem_sbt -= bttosbt(bt);
} else {
/*
* If the delta is greater than the timer period then
* just reset our time base instead of trying to catch
* up.
*/
vlapic->timer_fire_bt = btnow;
VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu "
"usecs, period is %lu usecs - resetting time base",
bttosbt(bt) / SBT_1US,
bttosbt(vlapic->timer_period_bt) / SBT_1US);
}
bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
vlapic_callout_reset(vlapic, rem_sbt);
}
done:
VLAPIC_TIMER_UNLOCK(vlapic);
}
void
vlapic_icrtmr_write_handler(struct vlapic *vlapic)
{
struct LAPIC *lapic;
sbintime_t sbt;
uint32_t icr_timer;
VLAPIC_TIMER_LOCK(vlapic);
lapic = vlapic->apic_page;
icr_timer = lapic->icr_timer;
vlapic->timer_period_bt = vlapic->timer_freq_bt;
bintime_mul(&vlapic->timer_period_bt, icr_timer);
if (icr_timer != 0) {
binuptime(&vlapic->timer_fire_bt);
bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
sbt = bttosbt(vlapic->timer_period_bt);
vlapic_callout_reset(vlapic, sbt);
} else
callout_stop(&vlapic->callout);
VLAPIC_TIMER_UNLOCK(vlapic);
}
/*
* This function populates 'dmask' with the set of vcpus that match the
* addressing specified by the (dest, phys, lowprio) tuple.
*
* 'x2apic_dest' specifies whether 'dest' is interpreted as x2APIC (32-bit)
* or xAPIC (8-bit) destination field.
*/
static void
vlapic_calcdest(struct vm *vm, cpuset_t *dmask, uint32_t dest, bool phys,
bool lowprio, bool x2apic_dest)
{
struct vlapic *vlapic;
uint32_t dfr, ldr, ldest, cluster;
uint32_t mda_flat_ldest, mda_cluster_ldest, mda_ldest, mda_cluster_id;
cpuset_t amask;
int vcpuid;
if ((x2apic_dest && dest == 0xffffffff) ||
(!x2apic_dest && dest == 0xff)) {
/*
* Broadcast in both logical and physical modes.
*/
*dmask = vm_active_cpus(vm);
return;
}
if (phys) {
/*
* Physical mode: destination is APIC ID.
*/
CPU_ZERO(dmask);
vcpuid = vm_apicid2vcpuid(vm, dest);
amask = vm_active_cpus(vm);
if (vcpuid < vm_get_maxcpus(vm) && CPU_ISSET(vcpuid, &amask))
CPU_SET(vcpuid, dmask);
} else {
/*
* In the "Flat Model" the MDA is interpreted as an 8-bit wide
* bitmask. This model is only available in the xAPIC mode.
*/
mda_flat_ldest = dest & 0xff;
/*
* In the "Cluster Model" the MDA is used to identify a
* specific cluster and a set of APICs in that cluster.
*/
if (x2apic_dest) {
mda_cluster_id = dest >> 16;
mda_cluster_ldest = dest & 0xffff;
} else {
mda_cluster_id = (dest >> 4) & 0xf;
mda_cluster_ldest = dest & 0xf;
}
/*
* Logical mode: match each APIC that has a bit set
* in its LDR that matches a bit in the ldest.
*/
CPU_ZERO(dmask);
amask = vm_active_cpus(vm);
CPU_FOREACH_ISSET(vcpuid, &amask) {
vlapic = vm_lapic(vm_vcpu(vm, vcpuid));
dfr = vlapic->apic_page->dfr;
ldr = vlapic->apic_page->ldr;
if ((dfr & APIC_DFR_MODEL_MASK) ==
APIC_DFR_MODEL_FLAT) {
ldest = ldr >> 24;
mda_ldest = mda_flat_ldest;
} else if ((dfr & APIC_DFR_MODEL_MASK) ==
APIC_DFR_MODEL_CLUSTER) {
if (x2apic(vlapic)) {
cluster = ldr >> 16;
ldest = ldr & 0xffff;
} else {
cluster = ldr >> 28;
ldest = (ldr >> 24) & 0xf;
}
if (cluster != mda_cluster_id)
continue;
mda_ldest = mda_cluster_ldest;
} else {
/*
* Guest has configured a bad logical
* model for this vcpu - skip it.
*/
VLAPIC_CTR1(vlapic, "vlapic has bad logical "
"model %x - cannot deliver interrupt", dfr);
continue;
}
if ((mda_ldest & ldest) != 0) {
CPU_SET(vcpuid, dmask);
if (lowprio)
break;
}
}
}
}
static VMM_STAT(VLAPIC_IPI_SEND, "ipis sent from vcpu");
static VMM_STAT(VLAPIC_IPI_RECV, "ipis received by vcpu");
static void
vlapic_set_tpr(struct vlapic *vlapic, uint8_t val)
{
struct LAPIC *lapic = vlapic->apic_page;
if (lapic->tpr != val) {
VLAPIC_CTR2(vlapic, "vlapic TPR changed from %#x to %#x",
lapic->tpr, val);
lapic->tpr = val;
vlapic_update_ppr(vlapic);
}
}
static uint8_t
vlapic_get_tpr(struct vlapic *vlapic)
{
struct LAPIC *lapic = vlapic->apic_page;
return (lapic->tpr);
}
void
vlapic_set_cr8(struct vlapic *vlapic, uint64_t val)
{
uint8_t tpr;
if (val & ~0xf) {
vm_inject_gp(vlapic->vcpu);
return;
}
tpr = val << 4;
vlapic_set_tpr(vlapic, tpr);
}
uint64_t
vlapic_get_cr8(struct vlapic *vlapic)
{
uint8_t tpr;
tpr = vlapic_get_tpr(vlapic);
return (tpr >> 4);
}
static bool
vlapic_is_icr_valid(uint64_t icrval)
{
uint32_t mode = icrval & APIC_DELMODE_MASK;
uint32_t level = icrval & APIC_LEVEL_MASK;
uint32_t trigger = icrval & APIC_TRIGMOD_MASK;
uint32_t shorthand = icrval & APIC_DEST_MASK;
switch (mode) {
case APIC_DELMODE_FIXED:
if (trigger == APIC_TRIGMOD_EDGE)
return (true);
/*
* AMD allows a level assert IPI and Intel converts a level
* assert IPI into an edge IPI.
*/
if (trigger == APIC_TRIGMOD_LEVEL && level == APIC_LEVEL_ASSERT)
return (true);
break;
case APIC_DELMODE_LOWPRIO:
case APIC_DELMODE_SMI:
case APIC_DELMODE_NMI:
case APIC_DELMODE_INIT:
if (trigger == APIC_TRIGMOD_EDGE &&
(shorthand == APIC_DEST_DESTFLD ||
shorthand == APIC_DEST_ALLESELF))
return (true);
/*
* AMD allows a level assert IPI and Intel converts a level
* assert IPI into an edge IPI.
*/
if (trigger == APIC_TRIGMOD_LEVEL &&
level == APIC_LEVEL_ASSERT &&
(shorthand == APIC_DEST_DESTFLD ||
shorthand == APIC_DEST_ALLESELF))
return (true);
/*
* An level triggered deassert INIT is defined in the Intel
* Multiprocessor Specification and the Intel Software Developer
* Manual. Due to the MPS it's required to send a level assert
* INIT to a cpu and then a level deassert INIT. Some operating
* systems e.g. FreeBSD or Linux use that algorithm. According
* to the SDM a level deassert INIT is only supported by Pentium
* and P6 processors. It's always send to all cpus regardless of
* the destination or shorthand field. It resets the arbitration
* id register. This register is not software accessible and
* only required for the APIC bus arbitration. So, the level
* deassert INIT doesn't need any emulation and we should ignore
* it. The SDM also defines that newer processors don't support
* the level deassert INIT and it's not valid any more. As it's
* defined for older systems, it can't be invalid per se.
* Otherwise, backward compatibility would be broken. However,
* when returning false here, it'll be ignored which is the
* desired behaviour.
*/
if (mode == APIC_DELMODE_INIT &&
trigger == APIC_TRIGMOD_LEVEL &&
level == APIC_LEVEL_DEASSERT)
return (false);
break;
case APIC_DELMODE_STARTUP:
if (shorthand == APIC_DEST_DESTFLD ||
shorthand == APIC_DEST_ALLESELF)
return (true);
break;
case APIC_DELMODE_RR:
/* Only available on AMD! */
if (trigger == APIC_TRIGMOD_EDGE &&
shorthand == APIC_DEST_DESTFLD)
return (true);
break;
case APIC_DELMODE_RESV:
return (false);
default:
__assert_unreachable();
}
return (false);
}
int
vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
{
int i;
bool phys;
cpuset_t dmask, ipimask;
uint64_t icrval;
uint32_t dest, vec, mode, shorthand;
struct vcpu *vcpu;
struct vm_exit *vmexit;
struct LAPIC *lapic;
lapic = vlapic->apic_page;
lapic->icr_lo &= ~APIC_DELSTAT_PEND;
icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo;
if (x2apic(vlapic))
dest = icrval >> 32;
else
dest = icrval >> (32 + 24);
vec = icrval & APIC_VECTOR_MASK;
mode = icrval & APIC_DELMODE_MASK;
phys = (icrval & APIC_DESTMODE_LOG) == 0;
shorthand = icrval & APIC_DEST_MASK;
VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec);
switch (shorthand) {
case APIC_DEST_DESTFLD:
vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false, x2apic(vlapic));
break;
case APIC_DEST_SELF:
CPU_SETOF(vlapic->vcpuid, &dmask);
break;
case APIC_DEST_ALLISELF:
dmask = vm_active_cpus(vlapic->vm);
break;
case APIC_DEST_ALLESELF:
dmask = vm_active_cpus(vlapic->vm);
CPU_CLR(vlapic->vcpuid, &dmask);
break;
default:
__assert_unreachable();
}
/*
* Ignore invalid combinations of the icr.
*/
if (!vlapic_is_icr_valid(icrval)) {
VLAPIC_CTR1(vlapic, "Ignoring invalid ICR %016lx", icrval);
return (0);
}
/*
* ipimask is a set of vCPUs needing userland handling of the current
* IPI.
*/
CPU_ZERO(&ipimask);
switch (mode) {
case APIC_DELMODE_FIXED:
if (vec < 16) {
vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR,
false);
VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec);
return (0);
}
CPU_FOREACH_ISSET(i, &dmask) {
vcpu = vm_vcpu(vlapic->vm, i);
lapic_intr_edge(vcpu, vec);
vmm_stat_incr(vlapic->vcpu, VLAPIC_IPI_SEND, 1);
vmm_stat_incr(vcpu, VLAPIC_IPI_RECV, 1);
VLAPIC_CTR2(vlapic,
"vlapic sending ipi %d to vcpuid %d", vec, i);
}
break;
case APIC_DELMODE_NMI:
CPU_FOREACH_ISSET(i, &dmask) {
vcpu = vm_vcpu(vlapic->vm, i);
vm_inject_nmi(vcpu);
VLAPIC_CTR1(vlapic,
"vlapic sending ipi nmi to vcpuid %d", i);
}
break;
case APIC_DELMODE_INIT:
case APIC_DELMODE_STARTUP:
if (!vlapic->ipi_exit) {
if (!phys)
break;
i = vm_apicid2vcpuid(vlapic->vm, dest);
if (i >= vm_get_maxcpus(vlapic->vm) ||
i == vlapic->vcpuid)
break;
CPU_SETOF(i, &ipimask);
break;
}
CPU_COPY(&dmask, &ipimask);
break;
default:
return (1);
}
if (!CPU_EMPTY(&ipimask)) {
vmexit = vm_exitinfo(vlapic->vcpu);
vmexit->exitcode = VM_EXITCODE_IPI;
vmexit->u.ipi.mode = mode;
vmexit->u.ipi.vector = vec;
*vm_exitinfo_cpuset(vlapic->vcpu) = ipimask;
*retu = true;
}
return (0);
}
static void
vlapic_handle_init(struct vcpu *vcpu, void *arg)
{
struct vlapic *vlapic = vm_lapic(vcpu);
vlapic_reset(vlapic);
}
int
vm_handle_ipi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
{
struct vlapic *vlapic = vm_lapic(vcpu);
cpuset_t *dmask = vm_exitinfo_cpuset(vcpu);
uint8_t vec = vme->u.ipi.vector;
*retu = true;
switch (vme->u.ipi.mode) {
case APIC_DELMODE_INIT: {
cpuset_t active, reinit;
active = vm_active_cpus(vcpu_vm(vcpu));
CPU_AND(&reinit, &active, dmask);
if (!CPU_EMPTY(&reinit)) {
vm_smp_rendezvous(vcpu, reinit, vlapic_handle_init,
NULL);
}
vm_await_start(vcpu_vm(vcpu), dmask);
if (!vlapic->ipi_exit)
*retu = false;
break;
}
case APIC_DELMODE_STARTUP:
/*
* Ignore SIPIs in any state other than wait-for-SIPI
*/
*dmask = vm_start_cpus(vcpu_vm(vcpu), dmask);
if (CPU_EMPTY(dmask)) {
*retu = false;
break;
}
/*
* Old bhyve versions don't support the IPI
* exit. Translate it into the old style.
*/
if (!vlapic->ipi_exit) {
vme->exitcode = VM_EXITCODE_SPINUP_AP;
vme->u.spinup_ap.vcpu = CPU_FFS(dmask) - 1;
vme->u.spinup_ap.rip = vec << PAGE_SHIFT;
}
break;
default:
__assert_unreachable();
}
return (0);
}
void
vlapic_self_ipi_handler(struct vlapic *vlapic, uint64_t val)
{
int vec;
KASSERT(x2apic(vlapic), ("SELF_IPI does not exist in xAPIC mode"));
vec = val & 0xff;
lapic_intr_edge(vlapic->vcpu, vec);
vmm_stat_incr(vlapic->vcpu, VLAPIC_IPI_SEND, 1);
vmm_stat_incr(vlapic->vcpu, VLAPIC_IPI_RECV, 1);
VLAPIC_CTR1(vlapic, "vlapic self-ipi %d", vec);
}
int
vlapic_pending_intr(struct vlapic *vlapic, int *vecptr)
{
struct LAPIC *lapic = vlapic->apic_page;
int idx, i, bitpos, vector;
uint32_t *irrptr, val;
vlapic_update_ppr(vlapic);
if (vlapic->ops.pending_intr)
return ((*vlapic->ops.pending_intr)(vlapic, vecptr));
irrptr = &lapic->irr0;
for (i = 7; i >= 0; i--) {
idx = i * 4;
val = atomic_load_acq_int(&irrptr[idx]);
bitpos = fls(val);
if (bitpos != 0) {
vector = i * 32 + (bitpos - 1);
if (PRIO(vector) > PRIO(lapic->ppr)) {
VLAPIC_CTR1(vlapic, "pending intr %d", vector);
if (vecptr != NULL)
*vecptr = vector;
return (1);
} else
break;
}
}
return (0);
}
void
vlapic_intr_accepted(struct vlapic *vlapic, int vector)
{
struct LAPIC *lapic = vlapic->apic_page;
uint32_t *irrptr, *isrptr;
int idx, stk_top;
if (vlapic->ops.intr_accepted)
return ((*vlapic->ops.intr_accepted)(vlapic, vector));
/*
* clear the ready bit for vector being accepted in irr
* and set the vector as in service in isr.
*/
idx = (vector / 32) * 4;
irrptr = &lapic->irr0;
atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
isrptr = &lapic->isr0;
isrptr[idx] |= 1 << (vector % 32);
VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
/*
* Update the PPR
*/
vlapic->isrvec_stk_top++;
stk_top = vlapic->isrvec_stk_top;
if (stk_top >= ISRVEC_STK_SIZE)
panic("isrvec_stk_top overflow %d", stk_top);
vlapic->isrvec_stk[stk_top] = vector;
}
void
vlapic_svr_write_handler(struct vlapic *vlapic)
{
struct LAPIC *lapic;
uint32_t old, new, changed;
lapic = vlapic->apic_page;
new = lapic->svr;
old = vlapic->svr_last;
vlapic->svr_last = new;
changed = old ^ new;
if ((changed & APIC_SVR_ENABLE) != 0) {
if ((new & APIC_SVR_ENABLE) == 0) {
/*
* The apic is now disabled so stop the apic timer
* and mask all the LVT entries.
*/
VLAPIC_CTR0(vlapic, "vlapic is software-disabled");
VLAPIC_TIMER_LOCK(vlapic);
callout_stop(&vlapic->callout);
VLAPIC_TIMER_UNLOCK(vlapic);
vlapic_mask_lvts(vlapic);
} else {
/*
* The apic is now enabled so restart the apic timer
* if it is configured in periodic mode.
*/
VLAPIC_CTR0(vlapic, "vlapic is software-enabled");
if (vlapic_periodic_timer(vlapic))
vlapic_icrtmr_write_handler(vlapic);
}
}
}
int
vlapic_read(struct vlapic *vlapic, int mmio_access, uint64_t offset,
uint64_t *data, bool *retu)
{
struct LAPIC *lapic = vlapic->apic_page;
uint32_t *reg;
int i;
/* Ignore MMIO accesses in x2APIC mode */
if (x2apic(vlapic) && mmio_access) {
VLAPIC_CTR1(vlapic, "MMIO read from offset %#lx in x2APIC mode",
offset);
*data = 0;
goto done;
}
if (!x2apic(vlapic) && !mmio_access) {
/*
* XXX Generate GP fault for MSR accesses in xAPIC mode
*/
VLAPIC_CTR1(vlapic, "x2APIC MSR read from offset %#lx in "
"xAPIC mode", offset);
*data = 0;
goto done;
}
if (offset > sizeof(*lapic)) {
*data = 0;
goto done;
}
offset &= ~3;
switch(offset)
{
case APIC_OFFSET_ID:
*data = lapic->id;
break;
case APIC_OFFSET_VER:
*data = lapic->version;
break;
case APIC_OFFSET_TPR:
*data = vlapic_get_tpr(vlapic);
break;
case APIC_OFFSET_APR:
*data = lapic->apr;
break;
case APIC_OFFSET_PPR:
*data = lapic->ppr;
break;
case APIC_OFFSET_EOI:
*data = lapic->eoi;
break;
case APIC_OFFSET_LDR:
*data = lapic->ldr;
break;
case APIC_OFFSET_DFR:
*data = lapic->dfr;
break;
case APIC_OFFSET_SVR:
*data = lapic->svr;
break;
case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
i = (offset - APIC_OFFSET_ISR0) >> 2;
reg = &lapic->isr0;
*data = *(reg + i);
break;
case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
i = (offset - APIC_OFFSET_TMR0) >> 2;
reg = &lapic->tmr0;
*data = *(reg + i);
break;
case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
i = (offset - APIC_OFFSET_IRR0) >> 2;
reg = &lapic->irr0;
*data = atomic_load_acq_int(reg + i);
break;
case APIC_OFFSET_ESR:
*data = lapic->esr;
break;
case APIC_OFFSET_ICR_LOW:
*data = lapic->icr_lo;
if (x2apic(vlapic))
*data |= (uint64_t)lapic->icr_hi << 32;
break;
case APIC_OFFSET_ICR_HI:
*data = lapic->icr_hi;
break;
case APIC_OFFSET_CMCI_LVT:
case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
*data = vlapic_get_lvt(vlapic, offset);
#ifdef INVARIANTS
reg = vlapic_get_lvtptr(vlapic, offset);
KASSERT(*data == *reg, ("inconsistent lvt value at "
"offset %#lx: %#lx/%#x", offset, *data, *reg));
#endif
break;
case APIC_OFFSET_TIMER_ICR:
*data = lapic->icr_timer;
break;
case APIC_OFFSET_TIMER_CCR:
*data = vlapic_get_ccr(vlapic);
break;
case APIC_OFFSET_TIMER_DCR:
*data = lapic->dcr_timer;
break;
case APIC_OFFSET_SELF_IPI:
/*
* XXX generate a GP fault if vlapic is in x2apic mode
*/
*data = 0;
break;
case APIC_OFFSET_RRR:
default:
*data = 0;
break;
}
done:
VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data);
return 0;
}
int
vlapic_write(struct vlapic *vlapic, int mmio_access, uint64_t offset,
uint64_t data, bool *retu)
{
struct LAPIC *lapic = vlapic->apic_page;
uint32_t *regptr;
int retval;
KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE,
("vlapic_write: invalid offset %#lx", offset));
VLAPIC_CTR2(vlapic, "vlapic write offset %#lx, data %#lx",
offset, data);
if (offset > sizeof(*lapic))
return (0);
/* Ignore MMIO accesses in x2APIC mode */
if (x2apic(vlapic) && mmio_access) {
VLAPIC_CTR2(vlapic, "MMIO write of %#lx to offset %#lx "
"in x2APIC mode", data, offset);
return (0);
}
/*
* XXX Generate GP fault for MSR accesses in xAPIC mode
*/
if (!x2apic(vlapic) && !mmio_access) {
VLAPIC_CTR2(vlapic, "x2APIC MSR write of %#lx to offset %#lx "
"in xAPIC mode", data, offset);
return (0);
}
retval = 0;
switch(offset)
{
case APIC_OFFSET_ID:
lapic->id = data;
vlapic_id_write_handler(vlapic);
break;
case APIC_OFFSET_TPR:
vlapic_set_tpr(vlapic, data & 0xff);
break;
case APIC_OFFSET_EOI:
vlapic_process_eoi(vlapic);
break;
case APIC_OFFSET_LDR:
lapic->ldr = data;
vlapic_ldr_write_handler(vlapic);
break;
case APIC_OFFSET_DFR:
lapic->dfr = data;
vlapic_dfr_write_handler(vlapic);
break;
case APIC_OFFSET_SVR:
lapic->svr = data;
vlapic_svr_write_handler(vlapic);
break;
case APIC_OFFSET_ICR_LOW:
lapic->icr_lo = data;
if (x2apic(vlapic))
lapic->icr_hi = data >> 32;
retval = vlapic_icrlo_write_handler(vlapic, retu);
break;
case APIC_OFFSET_ICR_HI:
lapic->icr_hi = data;
break;
case APIC_OFFSET_CMCI_LVT:
case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
regptr = vlapic_get_lvtptr(vlapic, offset);
*regptr = data;
vlapic_lvt_write_handler(vlapic, offset);
break;
case APIC_OFFSET_TIMER_ICR:
lapic->icr_timer = data;
vlapic_icrtmr_write_handler(vlapic);
break;
case APIC_OFFSET_TIMER_DCR:
lapic->dcr_timer = data;
vlapic_dcr_write_handler(vlapic);
break;
case APIC_OFFSET_ESR:
vlapic_esr_write_handler(vlapic);
break;
case APIC_OFFSET_SELF_IPI:
if (x2apic(vlapic))
vlapic_self_ipi_handler(vlapic, data);
break;
case APIC_OFFSET_VER:
case APIC_OFFSET_APR:
case APIC_OFFSET_PPR:
case APIC_OFFSET_RRR:
case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
case APIC_OFFSET_TIMER_CCR:
default:
// Read only.
break;
}
return (retval);
}
static void
vlapic_reset(struct vlapic *vlapic)
{
struct LAPIC *lapic;
lapic = vlapic->apic_page;
bzero(lapic, sizeof(struct LAPIC));
lapic->id = vlapic_get_id(vlapic);
lapic->version = VLAPIC_VERSION;
lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT);
lapic->dfr = 0xffffffff;
lapic->svr = APIC_SVR_VECTOR;
vlapic_mask_lvts(vlapic);
vlapic_reset_tmr(vlapic);
lapic->dcr_timer = 0;
vlapic_dcr_write_handler(vlapic);
vlapic->svr_last = lapic->svr;
}
void
vlapic_init(struct vlapic *vlapic)
{
KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized"));
KASSERT(vlapic->vcpuid >= 0 &&
vlapic->vcpuid < vm_get_maxcpus(vlapic->vm),
("vlapic_init: vcpuid is not initialized"));
KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not "
"initialized"));
/*
* If the vlapic is configured in x2apic mode then it will be
* accessed in the critical section via the MSR emulation code.
*
* Therefore the timer mutex must be a spinlock because blockable
* mutexes cannot be acquired in a critical section.
*/
mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN);
callout_init(&vlapic->callout, 1);
vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
if (vlapic->vcpuid == 0)
vlapic->msr_apicbase |= APICBASE_BSP;
vlapic->ipi_exit = false;
vlapic_reset(vlapic);
}
void
vlapic_cleanup(struct vlapic *vlapic)
{
callout_drain(&vlapic->callout);
mtx_destroy(&vlapic->timer_mtx);
}
uint64_t
vlapic_get_apicbase(struct vlapic *vlapic)
{
return (vlapic->msr_apicbase);
}
int
vlapic_set_apicbase(struct vlapic *vlapic, uint64_t new)
{
if (vlapic->msr_apicbase != new) {
VLAPIC_CTR2(vlapic, "Changing APIC_BASE MSR from %#lx to %#lx "
"not supported", vlapic->msr_apicbase, new);
return (-1);
}
return (0);
}
void
vlapic_set_x2apic_state(struct vcpu *vcpu, enum x2apic_state state)
{
struct vlapic *vlapic;
struct LAPIC *lapic;
vlapic = vm_lapic(vcpu);
if (state == X2APIC_DISABLED)
vlapic->msr_apicbase &= ~APICBASE_X2APIC;
else
vlapic->msr_apicbase |= APICBASE_X2APIC;
/*
* Reset the local APIC registers whose values are mode-dependent.
*
* XXX this works because the APIC mode can be changed only at vcpu
* initialization time.
*/
lapic = vlapic->apic_page;
lapic->id = vlapic_get_id(vlapic);
if (x2apic(vlapic)) {
lapic->ldr = x2apic_ldr(vlapic);
lapic->dfr = 0;
} else {
lapic->ldr = 0;
lapic->dfr = 0xffffffff;
}
if (state == X2APIC_ENABLED) {
if (vlapic->ops.enable_x2apic_mode)
(*vlapic->ops.enable_x2apic_mode)(vlapic);
}
}
void
vlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys,
int delmode, int vec)
{
struct vcpu *vcpu;
bool lowprio;
int vcpuid;
cpuset_t dmask;
if (delmode != IOART_DELFIXED &&
delmode != IOART_DELLOPRI &&
delmode != IOART_DELEXINT) {
VM_CTR1(vm, "vlapic intr invalid delmode %#x", delmode);
return;
}
lowprio = (delmode == IOART_DELLOPRI);
/*
* We don't provide any virtual interrupt redirection hardware so
* all interrupts originating from the ioapic or MSI specify the
* 'dest' in the legacy xAPIC format.
*/
vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false);
CPU_FOREACH_ISSET(vcpuid, &dmask) {
vcpu = vm_vcpu(vm, vcpuid);
if (delmode == IOART_DELEXINT) {
vm_inject_extint(vcpu);
} else {
lapic_set_intr(vcpu, vec, level);
}
}
}
void
vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum)
{
/*
* Post an interrupt to the vcpu currently running on 'hostcpu'.
*
* This is done by leveraging features like Posted Interrupts (Intel)
* Doorbell MSR (AMD AVIC) that avoid a VM exit.
*
* If neither of these features are available then fallback to
* sending an IPI to 'hostcpu'.
*/
if (vlapic->ops.post_intr)
(*vlapic->ops.post_intr)(vlapic, hostcpu);
else
ipi_cpu(hostcpu, ipinum);
}
bool
vlapic_enabled(struct vlapic *vlapic)
{
struct LAPIC *lapic = vlapic->apic_page;
if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 &&
(lapic->svr & APIC_SVR_ENABLE) != 0)
return (true);
else
return (false);
}
static void
vlapic_set_tmr(struct vlapic *vlapic, int vector, bool level)
{
struct LAPIC *lapic;
uint32_t *tmrptr, mask;
int idx;
lapic = vlapic->apic_page;
tmrptr = &lapic->tmr0;
idx = (vector / 32) * 4;
mask = 1 << (vector % 32);
if (level)
tmrptr[idx] |= mask;
else
tmrptr[idx] &= ~mask;
if (vlapic->ops.set_tmr != NULL)
(*vlapic->ops.set_tmr)(vlapic, vector, level);
}
void
vlapic_reset_tmr(struct vlapic *vlapic)
{
int vector;
VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered");
for (vector = 0; vector <= 255; vector++)
vlapic_set_tmr(vlapic, vector, false);
}
void
vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys,
int delmode, int vector)
{
cpuset_t dmask;
bool lowprio;
KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
/*
* A level trigger is valid only for fixed and lowprio delivery modes.
*/
if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) {
VLAPIC_CTR1(vlapic, "Ignoring level trigger-mode for "
"delivery-mode %d", delmode);
return;
}
lowprio = (delmode == APIC_DELMODE_LOWPRIO);
vlapic_calcdest(vlapic->vm, &dmask, dest, phys, lowprio, false);
if (!CPU_ISSET(vlapic->vcpuid, &dmask))
return;
VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector);
vlapic_set_tmr(vlapic, vector, true);
}
#ifdef BHYVE_SNAPSHOT
static void
vlapic_reset_callout(struct vlapic *vlapic, uint32_t ccr)
{
/* The implementation is similar to the one in the
* `vlapic_icrtmr_write_handler` function
*/
sbintime_t sbt;
struct bintime bt;
VLAPIC_TIMER_LOCK(vlapic);
bt = vlapic->timer_freq_bt;
bintime_mul(&bt, ccr);
if (ccr != 0) {
binuptime(&vlapic->timer_fire_bt);
bintime_add(&vlapic->timer_fire_bt, &bt);
sbt = bttosbt(bt);
vlapic_callout_reset(vlapic, sbt);
} else {
/* even if the CCR was 0, periodic timers should be reset */
if (vlapic_periodic_timer(vlapic)) {
binuptime(&vlapic->timer_fire_bt);
bintime_add(&vlapic->timer_fire_bt,
&vlapic->timer_period_bt);
sbt = bttosbt(vlapic->timer_period_bt);
callout_stop(&vlapic->callout);
vlapic_callout_reset(vlapic, sbt);
}
}
VLAPIC_TIMER_UNLOCK(vlapic);
}
int
vlapic_snapshot(struct vm *vm, struct vm_snapshot_meta *meta)
{
int ret;
struct vcpu *vcpu;
struct vlapic *vlapic;
struct LAPIC *lapic;
uint32_t ccr;
uint16_t i, maxcpus;
KASSERT(vm != NULL, ("%s: arg was NULL", __func__));
ret = 0;
maxcpus = vm_get_maxcpus(vm);
for (i = 0; i < maxcpus; i++) {
vcpu = vm_vcpu(vm, i);
if (vcpu == NULL)
continue;
vlapic = vm_lapic(vcpu);
/* snapshot the page first; timer period depends on icr_timer */
lapic = vlapic->apic_page;
SNAPSHOT_BUF_OR_LEAVE(lapic, PAGE_SIZE, meta, ret, done);
SNAPSHOT_VAR_OR_LEAVE(vlapic->esr_pending, meta, ret, done);
SNAPSHOT_VAR_OR_LEAVE(vlapic->timer_freq_bt.sec,
meta, ret, done);
SNAPSHOT_VAR_OR_LEAVE(vlapic->timer_freq_bt.frac,
meta, ret, done);
/*
* Timer period is equal to 'icr_timer' ticks at a frequency of
* 'timer_freq_bt'.
*/
if (meta->op == VM_SNAPSHOT_RESTORE) {
vlapic->timer_period_bt = vlapic->timer_freq_bt;
bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
}
SNAPSHOT_BUF_OR_LEAVE(vlapic->isrvec_stk,
sizeof(vlapic->isrvec_stk),
meta, ret, done);
SNAPSHOT_VAR_OR_LEAVE(vlapic->isrvec_stk_top, meta, ret, done);
SNAPSHOT_BUF_OR_LEAVE(vlapic->lvt_last,
sizeof(vlapic->lvt_last),
meta, ret, done);
if (meta->op == VM_SNAPSHOT_SAVE)
ccr = vlapic_get_ccr(vlapic);
SNAPSHOT_VAR_OR_LEAVE(ccr, meta, ret, done);
if (meta->op == VM_SNAPSHOT_RESTORE &&
vlapic_enabled(vlapic) && lapic->icr_timer != 0) {
/* Reset the value of the 'timer_fire_bt' and the vlapic
* callout based on the value of the current count
* register saved when the VM snapshot was created.
* If initial count register is 0, timer is not used.
* Look at "10.5.4 APIC Timer" in Software Developer Manual.
*/
vlapic_reset_callout(vlapic, ccr);
}
}
done:
return (ret);
}
#endif
plain