#include #include #include #include #include #include #include #include #include #include #include #include #include "../arch/x86_64/limine.h" /* VM-Exit 处理器：从 VMCS 读取 exit reason 与 guest 状态，分发处理。 * * Phase 1.1：处理 HLT（终止 guest）、EPT violation（动态映射）、异常。 * Phase 1.3：增加 VMCALL hypercall 处理（读 guest GPR、分发、写返回值）。 * Phase 1.4：EPT violation 区分 RAM（按需分配 HPA）与 virtio-mmio * （路由到设备模拟寄存器读写，不映射真实内存）。 * Phase 1.5：guest 平台设备模拟（legacy 模式，配合 noapic/nolapic）： * - COM1 8250 UART（console=ttyS0 输出 + host 串口输入桥） * - 8259 PIC（IMR 跟踪，EOI/ICW 吞掉） * - 8254 PIT ch0（divisor 跟踪，按 guest 编程频率注入 IRQ0 tick） * - CPUID passthrough（屏蔽 VMX/hypervisor 位） * - 虚拟中断注入（pending queue + interrupt-window exiting） * - VMX preemption timer 周期 exit（轮询 host 串口 RX + tick） */ extern volatile struct limine_hhdm_request g_hhdm_request; extern void outb(u16 port, u8 value); extern u8 inb(u16 port); extern void vmx_vmresume_wrapper(void); /* serial.c 导出的 host 串口与 TSC 频率接口 */ extern int serial_try_read(void); extern u64 serial_tsc_per_ms(void); extern void serial_putc(char c); static u64 g_vmexit_count; static u64 g_linux_tracked_cr3; #define CR3_RING 16 static u64 g_cr3_ring[CR3_RING]; static u32 g_cr3_ring_n; /* ---- 插桩: VM-Exit 统计计数器 (热路径，仅 STAT_INC) ---- */ INSTR_STAT_DECL(vmexit_total); INSTR_STAT_DECL(ept_violation); INSTR_STAT_DECL(ext_irq); INSTR_STAT_DECL(cpuid_exit); INSTR_STAT_DECL(io_exit); INSTR_STAT_DECL(intr_window); INSTR_STAT_DECL(preempt_timer); INSTR_STAT_DECL(msr_exit); INSTR_STAT_DECL(pre_resume); static void log_exit_diagnostics(u64 reason, u64 qualification, u64 rip, u64 len) { log_hex64("[VMEXIT] reason=", reason); log_hex64("[VMEXIT] qual=", qualification); log_hex64("[VMEXIT] rip=", rip); log_hex64("[VMEXIT] ilen=", len); } /* 处理 HLT： * - self-test guest：终止（guest 完成测试） * - Linux guest idle：推进 RIP 并留在 guest，等 PIT/virtio IRQ。 * daemon PARK 走 UTSM_HCALL_PARK，不再用 HLT（idle=poll 会在 * cpu_idle_poll 里把 current 读成 NULL 并 panic）。 */ static int handle_hlt(u64 rip, u64 instr_len, int *out_resume) { *out_resume = 0; /* 三种情况默认 exit-to-host */ if (g_linux_guest_active) { vmx_vmcs_write(VMCS_GUEST_RIP, rip + instr_len); *out_resume = 1; return 0; } else if (g_xj380_guest_active) { /* OpenXJ380 guest: 推进 RIP 并立即继续执行（不 park 不 terminate） */ vmx_vmcs_write(VMCS_GUEST_RIP, rip + instr_len); *out_resume = 1; } else { /* self-test guest: 终止 */ log_info("[VMEXIT] HLT - guest halted"); } return 0; } /* ===== EPT violation: guest RAM 按需分配 ===== * * guest 首次访问某 GPA 页时触发 EPT violation。我们按需分配一个 * 4KB HPA 页并建立 GPA→HPA 映射（不再是错误的 1:1 恒等映射—— * 那会把 guest GPA 直接当 host 物理地址，可能与 UTSM 自身内存冲突）。 * * 已映射区域的访问权限不足（如先 R 映射后写）也走这里：升级为 RWX。 */ /* 按需为普通 guest RAM GPA 分配一个 HPA 页并映射。返回 0 成功。 */ static int ept_map_guest_ram_page(u64 gpa_page) { dkm_dma_buffer buf; if (dma_alloc_pages(1, EPT_PAGE_SIZE, 0, &buf) != 0) { log_error("[VMEXIT] OOM for guest RAM page"); return -1; } if (ept_map_range(gpa_page, buf.phys, EPT_PAGE_SIZE, EPT_RWX) != 0) { log_error("[VMEXIT] EPT map guest RAM page failed"); return -1; } return 0; } /* ===== 虚拟中断注入（legacy PIC 模式，vector = 0x30 + irq） ===== * * Linux 以 noapic/nolapic 启动时走 XT-PIC：master PIC 向量基址 0x30。 * 我们向 guest 注入的外部中断只能是这些 ISA IRQ 向量： * IRQ0 (0x30) PIT tick、IRQ4 (0x34) COM1、IRQ5 (0x35) virtio-blk、 * IRQ6 (0x36) virtio-net。 * * 注入条件：guest RFLAGS.IF=1 且 interruptibility=0 且 IMR 未 mask。 * 不满足时 arm CPU_BASED_INTR_WINDOW_EXITING，guest 一旦开中断 * 立即产生 interrupt-window exit，届时再注入。 */ #define IRQ_PENDING_MAX 16 static u8 g_pending_vectors[IRQ_PENDING_MAX]; static int g_pending_head, g_pending_tail; /* guest 8259 PIC 状态（master） */ static u8 g_pic_imr = 0xFF; /* 初始全 mask */ static u8 g_pic_vec_base = 0x30; /* ICW2；x86_64 Linux ISA 向量基址 */ static int g_pic_icw_state = 0; /* 0=就绪(OCW), 1=等ICW2, 2=等ICW3, 3=等ICW4 */ static int g_pic_icw_sngl = 0; static int g_pic_icw_need4 = 0; static u32 g_irq_skip_logs; static u32 g_irq_inject_logs; #define GUEST_INTR_STI (1ULL << 0) #define GUEST_INTR_MOVSS (1ULL << 1) void vmx_guest_queue_irq(u32 vector) { /* 合并连续相同向量（PIT tick 合并 / COM1 RX 去重） */ if (g_pending_tail != g_pending_head) { int last = (g_pending_tail + IRQ_PENDING_MAX - 1) % IRQ_PENDING_MAX; if (g_pending_vectors[last] == (u8)vector) return; } int next = (g_pending_tail + 1) % IRQ_PENDING_MAX; if (next == g_pending_head) return; /* 队列满：丢弃 */ g_pending_vectors[g_pending_tail] = (u8)vector; g_pending_tail = next; } static void maybe_inject_irq(void) { if (g_pending_head == g_pending_tail) return; u8 vec = g_pending_vectors[g_pending_head]; /* IMR：vector = ICW2 基址 + irq。基址未写入前按 0x30。 */ if (vec >= g_pic_vec_base && vec < (u8)(g_pic_vec_base + 8)) { if (g_pic_imr & (1u << (vec - g_pic_vec_base))) { /* Keep the IRQ queued. virtio-mmio is edge-like: dropping it * here loses the completion forever if the guest parked while * the line was still masked. */ if (g_irq_skip_logs < 8) { g_irq_skip_logs++; log_hex64("[IRQ] masked imr=", g_pic_imr); log_hex64("[IRQ] masked vec=", vec); } return; } } u64 rflags = vmx_vmcs_read(VMCS_GUEST_RFLAGS); u64 intr_st = vmx_vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); u64 cpu = vmx_vmcs_read(VMCS_CPU_BASED_VM_EXEC_CONTROL); /* External IRQ may be injected while virtual-NMI blocking is set. * Only STI-shadow and MOV-SS-shadow must delay delivery. Requiring * interruptibility==0 froze PIT after park: nanosleep never woke. */ if ((rflags & 0x200) && !(intr_st & (GUEST_INTR_STI | GUEST_INTR_MOVSS))) { vmx_vmcs_write(VMCS_VM_ENTRY_INTERRUPT_INFO, VM_ENTRY_INTR_INFO_VALID | (VM_ENTRY_INTR_TYPE_HW_IRQ << 8) | vec); g_pending_head = (g_pending_head + 1) % IRQ_PENDING_MAX; if (cpu & CPU_BASED_INTR_WINDOW_EXITING) { vmx_vmcs_write(VMCS_CPU_BASED_VM_EXEC_CONTROL, cpu & ~CPU_BASED_INTR_WINDOW_EXITING); } if (g_irq_inject_logs < 4) { g_irq_inject_logs++; log_hex64("[IRQ] inject vec=", vec); log_hex64("[IRQ] inject base=", g_pic_vec_base); log_hex64("[IRQ] inject intr=", intr_st); } } else { if (g_irq_skip_logs < 8) { g_irq_skip_logs++; log_hex64("[IRQ] window if/sti rflags=", rflags); log_hex64("[IRQ] window intr=", intr_st); } if (!(cpu & CPU_BASED_INTR_WINDOW_EXITING)) { vmx_vmcs_write(VMCS_CPU_BASED_VM_EXEC_CONTROL, cpu | CPU_BASED_INTR_WINDOW_EXITING); } } } /* ===== guest COM1 8250 UART 模拟 ===== * * console=ttyS0 的 I/O 端口 0x3F8-0x3FF。 * TX：THR 写 → host 串口输出（与 UTSM log 共用 QEMU stdio）。 * RX：host 串口输入 → g_guest_rx 环形缓冲 → guest 读 RBR； * 有数据时注入 IRQ4（vector 0x34）。 * 寄存器：DLL/DLM/IER/FCR/LCR/MCR/SCR 存储，LSR/IIR/MSR 按状态合成。 */ #define GUEST_RX_SIZE 256 static u8 g_guest_rx[GUEST_RX_SIZE]; static int g_rx_head, g_rx_tail; static u8 g_com1_dll, g_com1_dlm, g_com1_ier, g_com1_fcr; static u8 g_com1_lcr, g_com1_mcr, g_com1_scr; static u64 com1_io(u32 port, int is_in, u64 value) { u32 reg = port - 0x3F8; int dlab = (g_com1_lcr & 0x80) != 0; if (is_in) { switch (reg) { case 0: /* RBR / DLL */ if (dlab) return g_com1_dll; if (g_rx_head != g_rx_tail) { u8 b = g_guest_rx[g_rx_head]; g_rx_head = (g_rx_head + 1) % GUEST_RX_SIZE; return b; } return 0; case 1: return dlab ? g_com1_dlm : g_com1_ier; case 2: /* IIR：bit0=0 表示有中断；bits7:6=1 表示 FIFO 启用 */ if (g_rx_head != g_rx_tail) return 0xC4; /* RX data available */ return 0xC1; /* no interrupt pending */ case 3: return g_com1_lcr; case 4: return g_com1_mcr; case 5: /* LSR：bit0 DR，bit5 THRE，bit6 TEMT */ return ((g_rx_head != g_rx_tail) ? 0x01 : 0x00) | 0x20 | 0x40; case 6: return 0xB0; /* MSR：CTS|DSR|DCD */ case 7: return g_com1_scr; } return 0xFF; } switch (reg) { case 0: if (dlab) g_com1_dll = (u8)value; else { /* Direct host UART write. serial_putc() waits on the same * COM1 LSR and can stall guest printk under file-backed serial. */ char c = (char)(value & 0xFF); if (c == '\n') { for (int i = 0; i < 10000 && !(inb(0x3F8 + 5) & 0x20); i++) {} outb(0x3F8, '\r'); } for (int i = 0; i < 10000 && !(inb(0x3F8 + 5) & 0x20); i++) {} outb(0x3F8, (u8)c); } break; case 1: if (dlab) g_com1_dlm = (u8)value; else g_com1_ier = (u8)value; break; case 2: g_com1_fcr = (u8)value; break; case 3: g_com1_lcr = (u8)value; break; case 4: g_com1_mcr = (u8)value; break; case 7: g_com1_scr = (u8)value; break; } return 0; } /* ===== guest 8254 PIT ch0 模拟（仅跟踪 divisor，tick 由 preemption exit 驱动） ===== */ static u16 g_pit_divisor = 0; /* 0 = 65536（默认 18.2Hz） */ static int g_pit_load_state = 0; /* 0=等 lo 字节, 1=等 hi 字节 */ static u8 g_pit_lo = 0; static u64 g_pit_period_tsc = 0; /* 每 tick 的 TSC 周期 */ static u64 g_pit_accum = 0; static u64 g_last_tick_tsc = 0; static u64 rdtsc_local(void) { u32 lo, hi; __asm__ volatile("rdtsc" : "=a"(lo), "=d"(hi)); return ((u64)hi << 32) | lo; } static void pit_recompute_period(void) { u64 tpm = serial_tsc_per_ms(); if (tpm == 0) tpm = 2000000; /* 未校准时按 ~2GHz 兜底 */ u64 div = g_pit_divisor ? g_pit_divisor : 65536; /* period_tsc = div / 1193182 Hz，用 ms 刻度换算避免溢出 */ g_pit_period_tsc = (div * tpm) / 1193; /* ≈ div * tpm * 1000 / 1193182 */ if (g_pit_period_tsc == 0) g_pit_period_tsc = 1; } static u64 pit_io(u32 port, int is_in, u64 value) { if (is_in) return 0; /* 计数器读回：恒 0（Linux 用 TSC clocksource，不依赖） */ if (port == 0x43) { /* mode/command：仅关心 ch0 + lo/hi 访问方式 */ if ((value & 0xC0) == 0x00 && (value & 0x30) == 0x30) { g_pit_load_state = 0; } return 0; } if (port == 0x40) { if (g_pit_load_state == 0) { g_pit_lo = (u8)value; g_pit_load_state = 1; } else { g_pit_divisor = (u16)(g_pit_lo | ((u16)(u8)value << 8)); g_pit_load_state = 0; pit_recompute_period(); } return 0; } return 0; /* 0x41/0x42 吞掉 */ } /* ===== 其余 legacy 端口（吞掉/返回安全值） ===== */ static u64 misc_io(u32 port, int is_in, u64 value, int *out_terminate) { /* 8259 PIC master */ if (port == 0x20) { if (!is_in) { if (value & 0x10) { /* ICW1 */ g_pic_icw_sngl = (value & 0x02) != 0; g_pic_icw_need4 = (value & 0x01) != 0; g_pic_icw_state = 1; g_pic_imr = 0xFF; } /* 其余 OCW2/OCW3（含 EOI）吞掉 */ } return 0; } if (port == 0x21) { if (is_in) return g_pic_imr; switch (g_pic_icw_state) { case 1: /* ICW2 向量基址（低 3 位忽略） */ g_pic_vec_base = (u8)value & 0xF8u; if (g_pic_vec_base == 0) g_pic_vec_base = 0x30; g_pic_icw_state = g_pic_icw_sngl ? (g_pic_icw_need4 ? 3 : 0) : 2; break; case 2: /* ICW3 */ g_pic_icw_state = g_pic_icw_need4 ? 3 : 0; break; case 3: /* ICW4 */ g_pic_icw_state = 0; break; default: /* OCW1 = IMR */ g_pic_imr = (u8)value; maybe_inject_irq(); break; } return 0; } /* 8259 PIC slave：吞掉（不用） */ if (port == 0xA0 || port == 0xA1) return is_in ? 0xFF : 0; /* 8254 PIT */ if (port >= 0x40 && port <= 0x43) return pit_io(port, is_in, value); /* Port 0x61: NMI/speaker/PIT2. Linux pit_hpet_ptimer_calibrate_cpu * waits for bit5 (PIT ch2 OUT). A constant 0 livelocks that loop. */ if (port == 0x61) { static u8 port61_gate; if (is_in) { port61_gate ^= 0x10; /* bit4 refresh toggle */ return (u8)((port61_gate & 0x13) | 0x20); /* bit5 = PIT2 OUT */ } port61_gate = (u8)((port61_gate & ~0x03) | (value & 0x03)); return 0; } /* PS/2 键盘/鼠标（i8042）：status 读 0（空），data 读 0，写吞掉。 * i8042 自检将失败 → guest 无 PS/2 键鼠（console 输入走串口）。 */ if (port == 0x60 || port == 0x64) return is_in ? 0 : 0; /* CMOS/RTC */ if (port == 0x70 || port == 0x71) return is_in ? 0 : 0; /* PCI config mechanism #1：对 guest 隐藏整个 PCI 总线。 * guest reconfig BAR 会破坏 UTSM 的 AHCI/e1000，故全部返回"无设备"。 */ if (port >= 0xCF8 && port <= 0xCFF) return is_in ? 0xFFFFFFFFULL : 0; /* 0xCF9 reset control：guest 请求重启 → 终止 guest 回 host */ if (port == 0xCF9) { if (!is_in) { log_warn("[VMEXIT] guest requested reboot via 0xCF9"); *out_terminate = 1; } return 0; } /* COM2-4：无设备 */ if ((port >= 0x2F8 && port <= 0x2FF) || (port >= 0x3E8 && port <= 0x3EF) || (port >= 0x2E8 && port <= 0x2EF)) { return is_in ? 0xFF : 0; } /* 其余端口（0x80 delay、0x61 NMI、0x92 A20、0xB2 APM、0xF0-0xFF FPU * error、0xED delay、ACPI 等）：读 0，写吞掉 */ return is_in ? 0 : 0; } /* ===== EXIT_IO_INSTRUCTION 处理 ===== * * exit qualification（Intel SDM 27.2.4.2? 实为 Vol 3D 表 27-6）： * bits 2:0 = 访问大小-1（0=1B, 1=2B, 3=4B） * bit 3 = 方向（0=OUT, 1=IN） * bit 4 = string 指令 * bit 5 = REP 前缀 * bits31:16 = 端口号 * OUT 数据在 guest RAX；IN 结果写回 guest RAX。 */ static int handle_io(u64 qualification, u64 rip, u64 instr_len, int *out_resume) { u32 port = (u32)(qualification >> 16); u32 size = (u32)(qualification & 7) + 1; int is_in = (qualification >> 3) & 1; u64 value = 0; int terminate = 0; if (!is_in) { value = g_guest_regs.rax; if (size == 1) value &= 0xFF; else if (size == 2) value &= 0xFFFF; else value &= 0xFFFFFFFFULL; } u64 result; if (port >= 0x3F8 && port <= 0x3FF) { result = com1_io(port, is_in, value); } else { result = misc_io(port, is_in, value, &terminate); } if (is_in) { if (size == 1) { g_guest_regs.rax = (g_guest_regs.rax & ~0xFFULL) | (result & 0xFF); } else if (size == 2) { g_guest_regs.rax = (g_guest_regs.rax & ~0xFFFFULL) | (result & 0xFFFF); } else { g_guest_regs.rax = result & 0xFFFFFFFFULL; /* 32 位写清零高 32 位 */ } } if (terminate) { *out_resume = 0; return 0; } /* Nested KVM sometimes reports VM-exit instruction length 0 for IN/OUT. * Adding 0 leaves RIP on the same opcode and livelocks extract_kernel * earlyprintk (millions of exits at one RIP). Immediate-port encodings * are 2 bytes; DX-port encodings are 1. */ if (instr_len == 0) { instr_len = ((qualification >> 6) & 1) ? 2 : 1; } vmx_vmcs_write(VMCS_GUEST_RIP, rip + instr_len); *out_resume = 1; return 0; } /* ===== EXIT_CPUID 处理：host passthrough + 屏蔽 VMX/hypervisor ===== * * CPUID 在 VMX non-root 无条件 exit。必须真实执行并把结果写回 guest * rax/rbx/rcx/rdx，否则 guest 特性探测全错（原"skip"实现会崩 Linux）。 * leaf 1 ecx：清 bit5(VMX) 与 bit31(hypervisor)，让 guest 认为自己在裸机。 */ static void cpuid_host(u32 a, u32 c, u32 *oa, u32 *ob, u32 *oc, u32 *od) { __asm__ volatile("cpuid" : "=a"(*oa), "=b"(*ob), "=c"(*oc), "=d"(*od) : "a"(a), "c"(c)); } static int handle_cpuid(u64 rip, u64 instr_len, int *out_resume) { u32 a = (u32)g_guest_regs.rax; u32 c = (u32)g_guest_regs.rcx; u32 oa, ob, oc, od; cpuid_host(a, c, &oa, &ob, &oc, &od); if (a == 1) { oc &= ~(1u << 5); /* VMX */ oc &= ~(1u << 17); /* PCID: keep Linux on MOV-CR3 TLB flush */ oc &= ~(1u << 12); /* FMA: requires AVX */ oc &= ~(1u << 29); /* F16C */ oc &= ~(1u << 31); /* hypervisor present */ } linux_xsave_adjust_cpuid(a, c, &oa, &ob, &oc, &od, vmx_vmcs_read(VMCS_GUEST_CR4)); if (a == 7 && c != 0) { /* Hide CET_SSS and other 7.1+ bits; Linux 6.6 probes subleaf 1. */ oa = 0; ob = 0; oc = 0; od = 0; } if (a == 7 && c == 0) { /* Nested KVM: these insns #UD in the guest even when host CPUID * advertises them. INVPCID is used by __flush_tlb_all. */ ob &= ~(1u << 0); /* FSGSBASE */ ob &= ~(1u << 5); /* AVX2: keep IFUNC on SSE until YMM save is on */ ob &= ~(1u << 10); /* INVPCID */ ob &= ~(1u << 16); /* AVX512F */ ob &= ~(1u << 17); /* AVX512DQ */ ob &= ~(1u << 21); /* AVX512IFMA */ ob &= ~(1u << 26); /* AVX512PF */ ob &= ~(1u << 27); /* AVX512ER */ ob &= ~(1u << 28); /* AVX512CD */ ob &= ~(1u << 30); /* AVX512BW */ ob &= ~(1u << 31); /* AVX512VL */ oc &= ~(1u << 1); /* AVX512VBMI */ oc &= ~(1u << 5); /* WAITPKG / TPAUSE */ oc &= ~(1u << 6); /* AVX512VBMI2 */ oc &= ~(1u << 7); /* CET_SS: SSP=0 makes PUSH fault at address 0 */ oc &= ~(1u << 11); /* AVX512VNNI */ oc &= ~(1u << 12); /* AVX512BITALG */ oc &= ~(1u << 14); /* AVX512VPOPCNTDQ */ oc &= ~(1u << 16); /* LA57 */ od &= ~(1u << 2); /* AVX5124VNNIW */ od &= ~(1u << 3); /* AVX5124FMAPS */ od &= ~(1u << 8); /* AVX512VP2INTERSECT */ od &= ~(1u << 20); /* CET_IBT */ od &= ~(1u << 23); /* AVX512FP16 */ } /* leaf 0x15/0x16 passthrough：QEMU 提供 TSC/总线频率， * guest 据此校准 TSC（绕过 PIT ch2 校准路径）。 */ g_guest_regs.rax = oa; g_guest_regs.rbx = ob; g_guest_regs.rcx = oc; g_guest_regs.rdx = od; vmx_vmcs_write(VMCS_GUEST_RIP, rip + instr_len); *out_resume = 1; return 0; } /* ===== resume 前例行工作：RX 轮询 + PIT tick + 注入 pending IRQ ===== */ static void poll_host_serial_to_guest(void) { for (int i = 0; i < 32; i++) { int b = serial_try_read(); if (b < 0) break; int next = (g_rx_tail + 1) % GUEST_RX_SIZE; if (next == g_rx_head) break; /* ring 满 */ g_guest_rx[g_rx_tail] = (u8)b; g_rx_tail = next; } if (g_rx_head != g_rx_tail) { vmx_guest_queue_irq(0x34); /* COM1 = IRQ4 → vector 0x34 */ } } static void pit_tick_update(void) { u64 now = rdtsc_local(); if (g_last_tick_tsc == 0) { g_last_tick_tsc = now; if (g_pit_period_tsc == 0) pit_recompute_period(); return; } u64 delta = now - g_last_tick_tsc; g_last_tick_tsc = now; if (g_pit_period_tsc == 0) return; g_pit_accum += delta; int n = 0; while (g_pit_accum >= g_pit_period_tsc && n < 8) { g_pit_accum -= g_pit_period_tsc; vmx_guest_queue_irq(g_pic_vec_base); /* PIT = IRQ0 */ n++; } if (g_pit_accum >= g_pit_period_tsc) g_pit_accum = 0; /* 防积压 */ } static u64 g_preemption_quantum = 0; #define VMX_PREEMPTION_INTERVAL_MS 10ULL /* ~80ms of guest time, then return to DSK so blit/Esc can run without * requiring the exec daemon to HLT (HLT freezes virtio completions). */ #define LINUX_HOST_SLICE_TICKS 8u static unsigned g_linux_slice_ticks; static int g_linux_timeslice_armed; static void vmexit_before_resume(void); void vmx_linux_timeslice_reset(void) { g_linux_slice_ticks = 0; g_linux_timeslice_armed = 1; } void vmx_linux_timeslice_disarm(void) { g_linux_timeslice_armed = 0; } void vmx_linux_prepare_entry(void) { vmexit_before_resume(); } u64 vmx_preemption_quantum(void) { if (g_preemption_quantum) return g_preemption_quantum; /* IA32_VMX_MISC[4:0] is the VMX-preemption-timer rate. The timer * decrements once per 2^rate TSC ticks; IA32_VMX_BASIC[55:48] is not * this rate (it contains unrelated VMX-basic capabilities). */ u64 misc = vmx_read_msr(IA32_VMX_MISC); u64 rate = misc & 0x1F; u64 tpm = serial_tsc_per_ms(); if (tpm == 0) tpm = 2000000; g_preemption_quantum = (tpm * VMX_PREEMPTION_INTERVAL_MS) >> rate; if (g_preemption_quantum == 0) g_preemption_quantum = 1; log_hex64("[VMX] preemption timer rate=", rate); log_hex64("[VMX] preemption timer interval_ms=", VMX_PREEMPTION_INTERVAL_MS); log_hex64("[VMX] preemption timer quantum=", g_preemption_quantum); return g_preemption_quantum; } int vmx_preemption_timer_supported(void) { u64 msr = vmx_read_msr(IA32_VMX_TRUE_PINBASED_CTLS); return (msr & (1ULL << (6 + 32))) != 0; /* bit6 允许置 1 */ } /* 每次 vmresume 前调用（vmexit_dispatch resume=1 路径末尾）。 * 重新 arm preemption timer（1ms 周期 exit），轮询 host 串口 RX， * 推进 PIT tick，注入 pending IRQ。 * 同时轮询 virtio-net RX：guest 投满 RX buffer 后不再 notify， * host 网卡收到的包靠这里周期性填充并注入 IRQ6。 */ static void linux_clear_guest_cet(void) { u64 cr4 = vmx_vmcs_read(VMCS_GUEST_CR4); if (cr4 & CR4_CET) { vmx_vmcs_write(VMCS_GUEST_CR4, cr4 & ~CR4_CET); } } static u64 g_pf_cr2; static int g_pf_need_cr2; static void write_cr2_local(u64 v) { __asm__ volatile("mov %0, %%cr2" :: "r"(v)); } static void vmexit_before_resume(void) { if (!g_linux_guest_active) return; if (g_pf_need_cr2) { write_cr2_local(g_pf_cr2); g_pf_need_cr2 = 0; } linux_clear_guest_cet(); linux_cet_vmcs_sync_host(); linux_cet_force_guest_off(); vmx_vmcs_write(VMCS_GUEST_PREEMPTION_TIMER, vmx_preemption_quantum()); poll_host_serial_to_guest(); virtio_net_poll(); virtio_input_poll(); /* VSCode Phase 2: drain pending kbd/mouse events */ pit_tick_update(); maybe_inject_irq(); } /* ===== EPT violation: 指令解码（用于 virtio-mmio MMIO 数据） ===== * * virtio-mmio 寄存器访问是普通 MOV 指令，如: * mov eax, [gpa] 读 (8A /r 或 A1) * mov [gpa], eax 写 (89 /r 或 A3) * movzx/movsx 等 * * 我们不需要完整解码——只需确定: * 1. 访问方向（读 or 写） * 2. 访问宽度（1/2/4/8 字节） * 3. 写访问时的写入数据；读访问时数据写到哪个 guest GPR * * 这通过解码指令字节 + 读 guest RIP 处的机器码完成。 * 用 GVA=EPT GPA 简化：guest 采用恒等页表，GUEST_LINEAR_ADDR 即 GPA。 */ /* Guest GVA → GPA via a CR3 value (GPA of the PGD). Linux PTI uses two * adjacent 4K PGDs; user and kernel CR3 differ by bit 12. */ static u64 guest_walk_cr3_at(u64 gva, u64 cr3, int five_level) { int first = five_level ? 0 : 1; int shifts[5] = { 48, 39, 30, 21, 12 }; u64 table_gpa = cr3 & 0x000FFFFFFFFFF000ULL; u64 hhdm = g_hhdm_request.response ? g_hhdm_request.response->offset : 0; if (!hhdm || !table_gpa) return 0; for (int level = first; level < 5; level++) { u64 table_hpa = ept_gpa_to_hpa(table_gpa); if (!table_hpa) return 0; u64 *table = (u64 *)(hhdm + table_hpa); u64 entry = table[(gva >> shifts[level]) & 0x1FF]; if (!(entry & 1)) return 0; if (shifts[level] == 30 && (entry & (1ULL << 7))) { return (entry & 0x000FFFFFC0000000ULL) + (gva & 0x3FFFFFFFULL); } if (shifts[level] == 21 && (entry & (1ULL << 7))) { return (entry & 0x000FFFFFFFE00000ULL) + (gva & 0x1FFFFFULL); } table_gpa = entry & 0x000FFFFFFFFFF000ULL; } return table_gpa + (gva & 0xFFFULL); } static u64 guest_current_cr3(void) { if (g_linux_tracked_cr3) return g_linux_tracked_cr3; return vmx_vmcs_read(VMCS_GUEST_CR3); } static int bytes_nonzero(const u8 *buf, u64 len) { u64 i; for (i = 0; i < len; i++) { if (buf[i]) return 1; } return 0; } static void track_guest_cr3(u64 val) { u32 i; g_linux_tracked_cr3 = val; vmx_vmcs_write(VMCS_GUEST_CR3, val); for (i = 0; i < g_cr3_ring_n && i < CR3_RING; i++) { if (g_cr3_ring[i] == val) return; } if (g_cr3_ring_n < CR3_RING) { g_cr3_ring[g_cr3_ring_n++] = val; } else { for (i = 1; i < CR3_RING; i++) g_cr3_ring[i - 1] = g_cr3_ring[i]; g_cr3_ring[CR3_RING - 1] = val; } /* MOV CR3 did not execute; flush the guest VPID so L2 does not keep * user translations from the previous mm. */ ept_flush_vpid(2); } static u64 guest_walk_cr3(u64 gva, int five_level) { u64 cr3 = guest_current_cr3(); u64 gpa = guest_walk_cr3_at(gva, cr3, five_level); if (gpa) return gpa; /* PTI: the other PGD is the adjacent 4K page. */ return guest_walk_cr3_at(gva, cr3 ^ 0x1000ULL, five_level); } u64 vmx_guest_gva_to_gpa(u64 gva) { u64 cr4 = vmx_vmcs_read(VMCS_GUEST_CR4); int la57 = (cr4 & (1ULL << 12)) ? 1 : 0; u64 gpa = guest_walk_cr3(gva, la57); if (gpa) return gpa; /* Linux 6.x L4 direct map. Kernel stacks live here after paging_init. */ if (gva >= 0xFFFF888000000000ULL && gva < 0xFFFFC88000000000ULL) { gpa = gva - 0xFFFF888000000000ULL; if (ept_gpa_to_hpa(gpa)) return gpa; } /* Retry the other paging mode if CR4.LA57 does not match the tables. */ gpa = guest_walk_cr3(gva, !la57); if (gpa) return gpa; if (ept_gpa_to_hpa(gva)) return gva; return 0; } /* 从 guest 内存读指令字节（GVA→GPA→HPA）。 * 若当前 CR3 走到全 0 页（PTI 走错 PGD / zeropage），改走 CR3^0x1000。 */ static int guest_fetch_at_cr3(u64 gva, u8 *buf, u64 len, u64 cr3) { u64 hhdm = g_hhdm_request.response ? g_hhdm_request.response->offset : 0; u64 cr4 = vmx_vmcs_read(VMCS_GUEST_CR4); int la57 = (cr4 & (1ULL << 12)) ? 1 : 0; if (!hhdm) return -1; for (u64 i = 0; i < len; i++) { u64 gpa = guest_walk_cr3_at(gva + i, cr3, la57); if (!gpa) gpa = guest_walk_cr3_at(gva + i, cr3, !la57); u64 hpa = gpa ? ept_gpa_to_hpa(gpa) : 0; if (hpa == 0) return -1; buf[i] = *(volatile u8 *)(hhdm + hpa); } return 0; } static int guest_fetch(u64 gva, u8 *buf, u64 len) { u64 cr3s[CR3_RING + 4]; u32 n = 0, i, j; u64 cur = guest_current_cr3(); u64 vmcs = vmx_vmcs_read(VMCS_GUEST_CR3); cr3s[n++] = cur; if (vmcs != cur) cr3s[n++] = vmcs; cr3s[n++] = cur ^ 0x1000ULL; for (i = g_cr3_ring_n; i > 0; i--) { int dup = 0; u64 c = g_cr3_ring[i - 1]; for (j = 0; j < n; j++) { if (cr3s[j] == c) { dup = 1; break; } } if (!dup) cr3s[n++] = c; } for (i = 0; i < n; i++) { if (guest_fetch_at_cr3(gva, buf, len, cr3s[i]) != 0) continue; if (bytes_nonzero(buf, len)) return 0; } /* Low identity window: PTI walk can hit the zeropage while the * CPU still executes the mapped ELF (busybox at 0x400000). */ if (gva < 0x04000000ULL && len > 0) { u64 hhdm = g_hhdm_request.response ? g_hhdm_request.response->offset : 0; if (hhdm) { int ok = 1; for (i = 0; i < len; i++) { u64 hpa = ept_gpa_to_hpa(gva + i); if (!hpa) { ok = 0; break; } buf[i] = *(volatile u8 *)(hhdm + hpa); } if (ok && bytes_nonzero(buf, len)) return 0; } } return -1; } /* 解码 MMIO 访问指令。 * rip_gpa : guest RIP（GPA，恒等映射） * is_write_out : 输出 1=写 MMIO，0=读 MMIO * width_out : 输出访问宽度（字节） * reg_field_out: 输出目标/源 GPR 在 g_guest_regs 中的索引（0-15） * 返回指令总长度（用于回退时推进 RIP），失败返回 0。 */ static u64 decode_mmio_insn(u64 rip_gpa, int *is_write_out, int *width_out, int *reg_field_out) { u8 insn[16]; if (guest_fetch(rip_gpa, insn, sizeof(insn)) != 0) return 0; u64 off = 0; int rex_w = 0, rex_r = 0; /* 跳过前缀 + REX */ for (;;) { u8 b = insn[off]; if (b == 0x66 || b == 0x67 || b == 0xF0 || b == 0xF2 || b == 0xF3 || (b >= 0x2E && b <= 0x3E && (b==0x2E||b==0x36||b==0x3E||b==0x26||b==0x64||b==0x65))) { off++; continue; } if (b >= 0x40 && b <= 0x4F) { /* REX */ rex_w = (b >> 3) & 1; rex_r = (b >> 2) & 1; off++; } break; } u8 op = insn[off]; u64 payload = off + 1; if (op == 0x0F) payload = off + 2; u8 modrm = insn[payload]; int mod = (modrm >> 6) & 3; int rm = modrm & 7; int reg = ((modrm >> 3) & 7) | (rex_r << 3); u64 n = payload + 1; /* opcode(s) + ModRM */ if (mod != 3 && rm == 4) { u8 sib = insn[n]; n += 1; if (mod == 0 && (sib & 7) == 5) n += 4; } if (mod == 1) n += 1; else if (mod == 2) n += 4; else if (mod == 0 && rm == 5) n += 4; /* mov r, m（读 MMIO）: 8A(8位) / 8B(全宽) */ if (op == 0x8A || op == 0x8B) { *is_write_out = 0; *width_out = (op == 0x8A) ? 1 : (rex_w ? 8 : 4); *reg_field_out = reg; return n; } /* mov m, r（写 MMIO）: 88(8位) / 89(全宽) */ if (op == 0x88 || op == 0x89) { *is_write_out = 1; *width_out = (op == 0x88) ? 1 : (rex_w ? 8 : 4); *reg_field_out = reg; return n; } /* movzx: 0F B6(8→宽) / 0F B7(16→宽) 读 */ if (op == 0x0F && (insn[off+1] == 0xB6 || insn[off+1] == 0xB7)) { *is_write_out = 0; *width_out = (insn[off+1] == 0xB6) ? 1 : 2; *reg_field_out = reg; return n; } return 0; } /* 按索引读写 guest GPR（用于 MMIO 数据的源/目的）。 */ static u64 *gpr_ptr(int idx) { switch (idx & 15) { case 0: return (u64 *)&g_guest_regs.rax; case 1: return (u64 *)&g_guest_regs.rcx; case 2: return (u64 *)&g_guest_regs.rdx; case 3: return (u64 *)&g_guest_regs.rbx; case 6: return (u64 *)&g_guest_regs.rsi; case 7: return (u64 *)&g_guest_regs.rdi; case 5: return (u64 *)&g_guest_regs.rbp; case 4: return (u64 *)&g_guest_regs.rax; /* rsp 不用作 MMIO 数据，占位 */ case 8: return (u64 *)&g_guest_regs.r8; case 9: return (u64 *)&g_guest_regs.r9; case 10: return (u64 *)&g_guest_regs.r10; case 11: return (u64 *)&g_guest_regs.r11; case 12: return (u64 *)&g_guest_regs.r12; case 13: return (u64 *)&g_guest_regs.r13; case 14: return (u64 *)&g_guest_regs.r14; case 15: return (u64 *)&g_guest_regs.r15; } return (u64 *)&g_guest_regs.rax; } static u64 read_gpr_full(int idx) { if ((idx & 15) == 4) return vmx_vmcs_read(VMCS_GUEST_RSP); return *gpr_ptr(idx); } static void write_gpr_full(int idx, u64 v) { if ((idx & 15) == 4) vmx_vmcs_write(VMCS_GUEST_RSP, v); else *gpr_ptr(idx) = v; } static int guest_write_u64(u64 gva, u64 value) { u64 hhdm = g_hhdm_request.response ? g_hhdm_request.response->offset : 0; u8 bytes[8]; u64 i; if (!hhdm) return -1; for (i = 0; i < 8; i++) bytes[i] = (u8)((value >> (i * 8)) & 0xFF); for (i = 0; i < 8; i++) { u64 gpa = vmx_guest_gva_to_gpa(gva + i); u64 hpa = gpa ? ept_gpa_to_hpa(gpa) : 0; if (!hpa && gpa) { ept_map_guest_ram_page(gpa & ~(EPT_PAGE_SIZE - 1)); hpa = ept_gpa_to_hpa(gpa); } if (!hpa) return -1; *(volatile u8 *)(hhdm + hpa) = bytes[i]; } return 0; } static int guest_read_u64(u64 gva, u64 *out) { u8 bytes[8]; u64 i, v = 0; if (guest_fetch(gva, bytes, 8) != 0) return -1; for (i = 0; i < 8; i++) v |= ((u64)bytes[i]) << (i * 8); *out = v; return 0; } static int guest_is_user_rip(u64 rip) { return (rip & (1ULL << 63)) == 0; } static u64 read_cr2_local(void) { u64 v; __asm__ volatile("mov %%cr2, %0" : "=r"(v)); return v; } /* Nested KVM leaks host SHSTK: CALL/RET/PUSH fault at SSP=0 (GVA 0). * Complete the data-stack side and skip the shadow-stack store. */ static int cet_emulate_shstk_insn(u64 rip, u64 *ilen_hint) { u8 insn[16]; u64 off = 0; int rex_b = 0; u64 rsp, next; static u32 emu_logs; if (guest_fetch(rip, insn, sizeof(insn)) != 0) return -1; if (insn[0] == 0 && insn[1] == 0 && insn[2] == 0 && insn[3] == 0) return -1; while (off < 4 && ((insn[off] >= 0x40 && insn[off] <= 0x4F))) { rex_b = insn[off] & 1; off++; } if (off >= sizeof(insn)) return -1; rsp = vmx_vmcs_read(VMCS_GUEST_RSP); /* PUSH r64: 50+rd */ if (insn[off] >= 0x50 && insn[off] <= 0x57) { int rd = (insn[off] - 0x50) | (rex_b << 3); u64 val = read_gpr_full(rd); rsp -= 8; if (guest_write_u64(rsp, val) != 0) return -1; vmx_vmcs_write(VMCS_GUEST_RSP, rsp); next = rip + off + 1; vmx_vmcs_write(VMCS_GUEST_RIP, next); if (emu_logs < 12) { log_hex64("[VMEXIT] CET emulate PUSH r rip=", rip); emu_logs++; } *ilen_hint = off + 1; return 0; } /* POP r64: 58+rd — data stack only */ if (insn[off] >= 0x58 && insn[off] <= 0x5F) { int rd = (insn[off] - 0x58) | (rex_b << 3); u64 val; if (guest_read_u64(rsp, &val) != 0) return -1; write_gpr_full(rd, val); vmx_vmcs_write(VMCS_GUEST_RSP, rsp + 8); vmx_vmcs_write(VMCS_GUEST_RIP, rip + off + 1); if (emu_logs < 12) { log_hex64("[VMEXIT] CET emulate POP r rip=", rip); emu_logs++; } return 0; } /* CALL rel32 */ if (insn[off] == 0xE8) { u32 rel = (u32)insn[off + 1] | ((u32)insn[off + 2] << 8) | ((u32)insn[off + 3] << 16) | ((u32)insn[off + 4] << 24); next = rip + off + 5; rsp -= 8; if (guest_write_u64(rsp, next) != 0) return -1; vmx_vmcs_write(VMCS_GUEST_RSP, rsp); vmx_vmcs_write(VMCS_GUEST_RIP, next + (u64)(int)rel); if (emu_logs < 12) { log_hex64("[VMEXIT] CET emulate CALL rel32 rip=", rip); emu_logs++; } return 0; } /* RET */ if (insn[off] == 0xC3) { u64 ret; if (guest_read_u64(rsp, &ret) != 0) return -1; vmx_vmcs_write(VMCS_GUEST_RSP, rsp + 8); vmx_vmcs_write(VMCS_GUEST_RIP, ret); if (emu_logs < 12) { log_hex64("[VMEXIT] CET emulate RET rip=", rip); emu_logs++; } return 0; } /* CALL/PUSH/JMP r/m64: FF /2, /6, /4 with ModRM.mod==3 */ if (insn[off] == 0xFF) { u8 modrm = insn[off + 1]; int mod = (modrm >> 6) & 3; int reg = (modrm >> 3) & 7; int rm = (modrm & 7) | (rex_b << 3); if (mod == 3 && (reg == 2 || reg == 6)) { u64 tgt = read_gpr_full(rm); next = rip + off + 2; rsp -= 8; if (guest_write_u64(rsp, next) != 0) return -1; vmx_vmcs_write(VMCS_GUEST_RSP, rsp); if (reg == 2) vmx_vmcs_write(VMCS_GUEST_RIP, tgt); else vmx_vmcs_write(VMCS_GUEST_RIP, next); if (emu_logs < 12) { log_hex64("[VMEXIT] CET emulate FF r/m rip=", rip); emu_logs++; } return 0; } } if (emu_logs < 12) { log_hex64("[VMEXIT] CET emulate miss rip=", rip); log_hex64("[VMEXIT] CET emulate miss insn=", ((u64)insn[0]) | ((u64)insn[1] << 8) | ((u64)insn[2] << 16) | ((u64)insn[3] << 24)); emu_logs++; } return -1; } /* 处理 virtio-mmio 设备 GPA 的访问：路由到设备模拟。返回 1=已处理（resume）。 */ static int handle_virtio_mmio_access(u64 gpa, u64 rip, int *out_resume) { int is_write, width, reg; u64 insn_len = decode_mmio_insn(rip, &is_write, &width, ®); if (insn_len == 0) { log_error("[VMEXIT] virtio-mmio: cannot decode access insn"); log_hex64("[VMEXIT] rip=", rip); *out_resume = 0; return -1; } u64 value = 0; if (is_write) { /* 从 guest GPR 取写入数据（按宽度截断） */ u64 raw = *gpr_ptr(reg); if (width == 1) value = raw & 0xFF; else if (width == 2) value = raw & 0xFFFF; else if (width == 4) value = raw & 0xFFFFFFFFULL; else value = raw; virtio_mmio_write(gpa, value, width); } else { /* 从设备读，写入 guest GPR（按宽度） */ value = virtio_mmio_read(gpa, width); u64 *dst = gpr_ptr(reg); if (width == 1) *dst = (*dst & ~0xFFULL) | (value & 0xFF); else if (width == 2) *dst = (*dst & ~0xFFFFULL) | (value & 0xFFFF); else if (width == 4) *dst = value & 0xFFFFFFFFULL; /* 32位写清零高32 */ else *dst = value; } /* 推进 RIP 越过 MMIO 指令，resume guest */ vmx_vmcs_write(VMCS_GUEST_RIP, rip + insn_len); *out_resume = 1; return 0; } /* 处理 EPT violation：区分 guest RAM（按需分配）与 virtio-mmio（设备模拟）。 * * exit qualification bits: * bit0 = read access, bit1 = write access, bit2 = instruction fetch * bit3 = EPT entry readable, bit4 = writable, bit5 = executable * virtio-mmio GPA 故意不映射 → bit3/4/5 全 0 → "访问不存在页"。 */ static int handle_ept_violation(u64 qualification, u64 rip, int *out_resume) { (void)qualification; /* virtio-mmio 判定只看 GPA，不看访问类型 */ u64 gpa = vmx_vmcs_read(VMCS_GUEST_PHYSICAL_ADDR); /* virtio-mmio 设备区：路由到设备模拟，绝不映射真实内存 */ if (IS_VIRTIO_MMIO_GPA(gpa)) { return handle_virtio_mmio_access(gpa, rip, out_resume); } /* 普通 guest RAM：按需分配 HPA 页并映射。 * Nested KVM can deliver spurious EPT violations on an already-mapped * GPA. Allocating a fresh page would discard libc/text that Linux * already wrote (PTE then points at a zero hole). Upgrade flags. */ u64 page = gpa & ~(EPT_PAGE_SIZE - 1); u64 existing = ept_gpa_to_hpa(page); if (existing) { static u32 reuse_logs; if (ept_map_range(page, existing, EPT_PAGE_SIZE, EPT_RWX) != 0) { log_error("[VMEXIT] EPT upgrade existing page failed"); log_hex64("[VMEXIT] gpa=", gpa); *out_resume = 0; return -1; } ept_flush_ept(); if (reuse_logs < 8) { reuse_logs++; log_hex64("[VMEXIT] EPT reuse gpa=", gpa); log_hex64("[VMEXIT] EPT reuse hpa=", existing); } *out_resume = 1; return 0; } if (ept_map_guest_ram_page(page) != 0) { log_error("[VMEXIT] EPT violation: guest RAM map failed"); log_hex64("[VMEXIT] gpa=", gpa); *out_resume = 0; return -1; } *out_resume = 1; return 0; } static void log_guest_page_walk(u64 linear) { u64 cr3 = guest_current_cr3(); u64 cr4 = vmx_vmcs_read(VMCS_GUEST_CR4); int shifts[5] = { 48, 39, 30, 21, 12 }; int first = (cr4 & (1ULL << 12)) ? 0 : 1; /* CR4.LA57 */ u64 table_gpa = cr3 & 0x000FFFFFFFFFF000ULL; u64 hhdm = g_hhdm_request.response ? g_hhdm_request.response->offset : 0; log_hex64("[VMEXIT] walk cr4=", cr4); for (int level = first; level < 5; level++) { u64 table_hpa = ept_gpa_to_hpa(table_gpa); log_hex64("[VMEXIT] walk table_gpa=", table_gpa); log_hex64("[VMEXIT] walk table_hpa=", table_hpa); if (!table_hpa || !hhdm) break; u64 *table = (u64 *)(hhdm + table_hpa); u64 index = (linear >> shifts[level]) & 0x1FF; u64 entry = table[index]; log_hex64("[VMEXIT] walk index=", index); log_hex64("[VMEXIT] walk entry=", entry); if (!(entry & 1)) break; if ((shifts[level] == 30 || shifts[level] == 21) && (entry & (1ULL << 7))) { break; } table_gpa = entry & 0x000FFFFFFFFFF000ULL; } } /* MOV CR3: keep a software copy. Nested KVM leaves VMCS GUEST_CR3 stuck * on an early kernel PGD (zero pages for user GVA). */ static int handle_cr_access(u64 qualification, u64 rip, u64 instr_len, int *out_resume) { u32 cr = (u32)(qualification & 0xF); u32 type = (u32)((qualification >> 4) & 3); u32 reg = (u32)((qualification >> 8) & 0xF); static u32 logs; if (cr == 3 && type == 0) { u64 val = read_gpr_full((int)reg); track_guest_cr3(val); if (logs < 12) { log_hex64("[VMEXIT] MOV CR3=", val); logs++; } } else if (cr == 3 && type == 1) { write_gpr_full((int)reg, guest_current_cr3()); } vmx_vmcs_write(VMCS_GUEST_RIP, rip + (instr_len ? instr_len : 3)); *out_resume = 1; return 0; } /* 处理通用异常：打印诊断信息，终止 guest。 */ static int handle_exception(u64 qualification, u64 rip, int *out_resume) { static u32 g_exc_logs; static u32 g_user_exc_logs; u64 intr_info = vmx_vmcs_read(VMCS_EXIT_INTR_INFO); int verbose = g_exc_logs < 8; int user_rip = guest_is_user_rip(rip); if (verbose) g_exc_logs++; if (verbose) { log_hex64("[VMEXIT] exception intr_info=", intr_info); if (intr_info & (1ULL << 11)) { log_hex64("[VMEXIT] exception error_code=", vmx_vmcs_read(VMCS_EXIT_INTR_ERROR_CODE)); } log_hex64("[VMEXIT] exception gla=", vmx_vmcs_read(VMCS_GUEST_LINEAR_ADDR)); log_hex64("[VMEXIT] exception qual=", qualification); log_hex64("[VMEXIT] exception rip=", rip); log_hex64("[VMEXIT] exception cr3=", vmx_vmcs_read(VMCS_GUEST_CR3)); log_hex64("[VMEXIT] exception rsp=", vmx_vmcs_read(VMCS_GUEST_RSP)); log_hex64("[VMEXIT] exception rbx=", g_guest_regs.rbx); log_hex64("[VMEXIT] exception rsi=", g_guest_regs.rsi); log_hex64("[VMEXIT] exception rdi=", g_guest_regs.rdi); log_hex64("[VMEXIT] exception r11=", g_guest_regs.r11); log_hex64("[VMEXIT] exception r15=", g_guest_regs.r15); { u8 insn[8]; if (guest_fetch(rip, insn, sizeof(insn)) == 0) { log_hex64("[VMEXIT] insn0=", ((u64)insn[0]) | ((u64)insn[1] << 8) | ((u64)insn[2] << 16) | ((u64)insn[3] << 24) | ((u64)insn[4] << 32) | ((u64)insn[5] << 40) | ((u64)insn[6] << 48) | ((u64)insn[7] << 56)); } } if ((intr_info & 0xFF) == 14) { log_guest_page_walk(qualification); } } if (g_linux_guest_active && user_rip && g_user_exc_logs < 16) { u8 insn[8]; g_user_exc_logs++; log_hex64("[VMEXIT] user-exc vec=", intr_info & 0xFF); log_hex64("[VMEXIT] user-exc err=", (intr_info & (1ULL << 11)) ? vmx_vmcs_read(VMCS_EXIT_INTR_ERROR_CODE) : 0); log_hex64("[VMEXIT] user-exc rip=", rip); log_hex64("[VMEXIT] user-exc cr3=", vmx_vmcs_read(VMCS_GUEST_CR3)); log_hex64("[VMEXIT] user-exc tracked=", guest_current_cr3()); log_hex64("[VMEXIT] user-exc csar=", vmx_vmcs_read(VMCS_GUEST_CS_ACCESS)); log_hex64("[VMEXIT] user-exc ilen=", vmx_vmcs_read(VMCS_INSTRUCTION_LENGTH)); log_hex64("[VMEXIT] user-exc walk=", vmx_guest_gva_to_gpa(rip)); log_hex64("[VMEXIT] user-exc hpa=", ept_gpa_to_hpa(vmx_guest_gva_to_gpa(rip))); { u64 gpa = vmx_guest_gva_to_gpa(rip); u64 hpa = gpa ? ept_gpa_to_hpa(gpa) : 0; u64 hhdm = g_hhdm_request.response ? g_hhdm_request.response->offset : 0; if (hpa && hhdm) { u8 *p = (u8 *)(hhdm + hpa); log_hex64("[VMEXIT] user-exc mem=", ((u64)p[0]) | ((u64)p[1] << 8) | ((u64)p[2] << 16) | ((u64)p[3] << 24) | ((u64)p[4] << 32) | ((u64)p[5] << 40) | ((u64)p[6] << 48) | ((u64)p[7] << 56)); } gpa = guest_walk_cr3_at(rip, guest_current_cr3() ^ 0x1000ULL, 0); hpa = gpa ? ept_gpa_to_hpa(gpa) : 0; log_hex64("[VMEXIT] user-exc xor-gpa=", gpa); log_hex64("[VMEXIT] user-exc xor-hpa=", hpa); if (hpa && hhdm) { u8 *p = (u8 *)(hhdm + hpa); log_hex64("[VMEXIT] user-exc xor-mem=", ((u64)p[0]) | ((u64)p[1] << 8) | ((u64)p[2] << 16) | ((u64)p[3] << 24) | ((u64)p[4] << 32) | ((u64)p[5] << 40) | ((u64)p[6] << 48) | ((u64)p[7] << 56)); } } if (guest_fetch(rip, insn, sizeof(insn)) == 0) { log_hex64("[VMEXIT] user-exc insn0=", ((u64)insn[0]) | ((u64)insn[1] << 8) | ((u64)insn[2] << 16) | ((u64)insn[3] << 24) | ((u64)insn[4] << 32) | ((u64)insn[5] << 40) | ((u64)insn[6] << 48) | ((u64)insn[7] << 56)); } else { log_info("[VMEXIT] user-exc fetch failed"); } if (g_user_exc_logs == 1) log_guest_page_walk(rip); } if ((intr_info & 0xFF) == 8) { log_error("[VMEXIT] #DF (will not reinject)"); log_hex64("[VMEXIT] df cr0=", vmx_vmcs_read(VMCS_GUEST_CR0)); log_hex64("[VMEXIT] df cr3=", vmx_vmcs_read(VMCS_GUEST_CR3)); log_hex64("[VMEXIT] df cr4=", vmx_vmcs_read(VMCS_GUEST_CR4)); log_hex64("[VMEXIT] df idtr=", vmx_vmcs_read(VMCS_GUEST_IDTR_BASE)); log_hex64("[VMEXIT] df idtl=", vmx_vmcs_read(VMCS_GUEST_IDTR_LIMIT)); *out_resume = 0; return 0; } if (g_linux_guest_active) { u32 vec = (u32)(intr_info & 0xFF); u32 err = 0; u64 ilen = vmx_vmcs_read(VMCS_INSTRUCTION_LENGTH); if (intr_info & (1ULL << 11)) err = (u32)vmx_vmcs_read(VMCS_EXIT_INTR_ERROR_CODE); /* User #PF at page 0 / wrap: leaked SHSTK store with SSP=0. * Always stash CR2 so a reinjected #PF still sees the L2 address * (CR2 is not a VMCS field). */ if (vec == 14) { u64 cr2 = read_cr2_local(); u64 gla = vmx_vmcs_read(VMCS_GUEST_LINEAR_ADDR); u64 addr = cr2 ? cr2 : gla; g_pf_cr2 = addr; g_pf_need_cr2 = 1; if (guest_is_user_rip(rip) && (addr < 0x1000ULL || addr >= 0xFFFFFFFFFFFFF000ULL)) { u64 dummy = 0; if (cet_emulate_shstk_insn(rip, &dummy) == 0) { g_pf_need_cr2 = 0; vmx_vmcs_write(VMCS_VM_ENTRY_INTERRUPT_INFO, 0); *out_resume = 1; return 0; } } } /* IBT: skip only when the bytes are ENDBR or the four-NOP strip. * A 16-byte-aligned zero-fetch skip desynced utsm_exec_daemon. */ if ((vec == 13 || vec == 6 || vec == 21) && guest_is_user_rip(rip) && (err == 0 || vec == 21)) { u8 insn[4]; int skip4 = 0; if (guest_fetch(rip, insn, 4) == 0) { if (insn[0] == 0xF3 && insn[1] == 0x0F && insn[2] == 0x1E && (insn[3] == 0xFA || insn[3] == 0xFB)) skip4 = 1; if (insn[0] == 0x90 && insn[1] == 0x90 && insn[2] == 0x90 && insn[3] == 0x90) skip4 = 1; } if (skip4) { static u32 endbr_skips; if (endbr_skips < 8) { log_hex64("[VMEXIT] skip landing-pad rip=", rip); log_hex64("[VMEXIT] skip landing-pad ilen=", ilen); endbr_skips++; } vmx_vmcs_write(VMCS_VM_ENTRY_INTERRUPT_INFO, 0); vmx_vmcs_write(VMCS_GUEST_RIP, rip + 4); *out_resume = 1; return 0; } } } if (g_linux_guest_active) { /* Linux's decompressor intentionally faults while extending its * identity map. Reinject the intercepted hardware exception with * its original error code so the guest IDT can service it. */ if (intr_info & (1ULL << 11)) { vmx_vmcs_write(VMCS_VM_ENTRY_EXCEPTION_ERROR, vmx_vmcs_read(VMCS_EXIT_INTR_ERROR_CODE)); } vmx_vmcs_write(VMCS_VM_ENTRY_INTERRUPT_INFO, intr_info); *out_resume = 1; return 0; } *out_resume = 0; return 0; } /* VM-Exit 主分发器。从 vmx_vm_exit_handler（汇编入口）调用。 * 返回 1 表示 vmresume；0 表示终止 guest。 */ int vmexit_dispatch(void) { u64 reason = vmx_vmcs_read(VMCS_EXIT_REASON) & 0xFFFF; u64 qualification = vmx_vmcs_read(VMCS_EXIT_QUALIFICATION); u64 rip = vmx_vmcs_read(VMCS_GUEST_RIP); u64 instr_len = vmx_vmcs_read(VMCS_INSTRUCTION_LENGTH); int resume = 0; if (g_linux_guest_active) linux_cet_restore_host(); g_vmexit_count++; INSTR_STAT_INC(vmexit_total); if (g_linux_guest_active && (g_vmexit_count <= 32 || (g_vmexit_count & 0xFFF) == 0)) { log_hex64("[VMEXIT] n=", g_vmexit_count); log_hex64("[VMEXIT] reason=", reason); log_hex64("[VMEXIT] rip=", rip); log_hex64("[VMEXIT] qual=", qualification); log_hex64("[VMEXIT] ilen=", instr_len); log_hex64("[VMEXIT] idtr=", vmx_vmcs_read(VMCS_GUEST_IDTR_BASE)); if (reason == EXIT_EPT_VIOLATION) { log_hex64("[VMEXIT] ept gpa=", vmx_vmcs_read(VMCS_GUEST_PHYSICAL_ADDR)); } } switch (reason) { case EXIT_HLT: handle_hlt(rip, instr_len, &resume); break; case EXIT_ENTRY_FAIL_MSR_LOADING: log_warn("[VMEXIT] VM-entry MSR-load failed; dropping CET MSR list"); vmx_vmcs_write(VMCS_VM_ENTRY_MSR_LOAD_COUNT, 0); resume = 1; break; case EXIT_EPT_VIOLATION: INSTR_STAT_INC(ept_violation); handle_ept_violation(qualification, rip, &resume); break; case EXIT_EPT_MISCONFIG: { /* Nested KVM: guest PAT (e.g. WC for virtio-gpu shmem) combined * with EPT WB is a reserved effective type → misconfig. Rewrite * the leaf with Ignore PAT and resume; do not kill the vCPU. */ u64 gpa = vmx_vmcs_read(VMCS_GUEST_PHYSICAL_ADDR); u64 page = gpa & ~(EPT_PAGE_SIZE - 1); static u32 misconfig_logs; INSTR_PROBE(EPTM, gpa, rip, 0, 0); if (misconfig_logs < 16) { misconfig_logs++; log_error("[VMEXIT] EPT misconfig"); log_hex64("[VMEXIT] gpa=", gpa); ept_log_walk(page); } if (IS_VIRTIO_MMIO_GPA(gpa)) { resume = 0; break; } if (ept_repair_leaf(page, EPT_RWX) != 0) { if (ept_map_guest_ram_page(page) != 0) { log_error("[VMEXIT] EPT misconfig repair failed"); resume = 0; break; } if (misconfig_logs <= 16) log_hex64("[VMEXIT] EPT misconfig replaced gpa=", page); } ept_flush_ept(); resume = 1; break; } case EXIT_EXCEPTION_NMI: INSTR_PROBE(EXCP, qualification, rip, 0, 0); handle_exception(qualification, rip, &resume); break; case EXIT_TRIPLE_FAULT: INSTR_PROBE(EXCP, 0, rip, 0, 0); log_error("[VMEXIT] triple fault"); log_hex64("[VMEXIT] triple rip=", rip); log_hex64("[VMEXIT] triple cr0=", vmx_vmcs_read(VMCS_GUEST_CR0)); log_hex64("[VMEXIT] triple cr3=", vmx_vmcs_read(VMCS_GUEST_CR3)); log_hex64("[VMEXIT] triple cr4=", vmx_vmcs_read(VMCS_GUEST_CR4)); log_hex64("[VMEXIT] triple idtr=", vmx_vmcs_read(VMCS_GUEST_IDTR_BASE)); log_hex64("[VMEXIT] triple idtl=", vmx_vmcs_read(VMCS_GUEST_IDTR_LIMIT)); resume = 0; break; case EXIT_EXTERNAL_INTERRUPT: INSTR_STAT_INC(ext_irq); /* 外部中断到达 host（真机比 QEMU 更频繁触发）。 * 当前 UTSM 不处理 host 外部中断（依赖 PIC/IDT）， * 直接 resume guest。真机上此 exit 由 PIN_EXT_INTERRUPT_EXITING * 触发，不应终止 guest。 */ resume = 1; break; case EXIT_INIT_SIGNAL: /* INIT 信号在 VMX root 下被阻断，但 non-root 会 exit。 * 真机 BSP 向 AP 发 INIT-IPI-SIPI 时可能触发。 * 吞掉：不执行 INIT，直接 resume。 */ resume = 1; break; case EXIT_INVD: /* INVD（cache flush）：吞掉，直接 resume。 */ vmx_vmcs_write(VMCS_GUEST_RIP, rip + instr_len); resume = 1; break; case EXIT_INVLPG: /* Guest TLB invalidate. EPT owns the real translation; skip. */ vmx_vmcs_write(VMCS_GUEST_RIP, rip + instr_len); resume = 1; break; case EXIT_CONTROL_ACCESS: handle_cr_access(qualification, rip, instr_len, &resume); break; case EXIT_MONITOR: case EXIT_MWAIT: /* MONITOR/MWAIT：在 noapic 模式下 Linux 不会主动使用， * 但某些驱动路径可能触发。直接 resume。 */ vmx_vmcs_write(VMCS_GUEST_RIP, rip + instr_len); resume = 1; break; case EXIT_CPUID: INSTR_STAT_INC(cpuid_exit); /* host passthrough + 屏蔽 VMX/hypervisor 位（handle_cpuid 内推进 RIP） */ handle_cpuid(rip, instr_len, &resume); break; case EXIT_XSETBV: /* XSETBV always VM-exits. Linux fpu__init writes XCR0 here. */ { u32 xcr = (u32)g_guest_regs.rcx; u64 val = ((u64)(u32)g_guest_regs.rdx << 32) | (u32)g_guest_regs.rax; if (xcr == 0) linux_xsave_set_guest_xcr0(val); vmx_vmcs_write(VMCS_GUEST_RIP, rip + (instr_len ? instr_len : 3)); resume = 1; } break; case EXIT_WBINVD: vmx_vmcs_write(VMCS_GUEST_RIP, rip + (instr_len ? instr_len : 2)); resume = 1; break; case EXIT_IO_INSTRUCTION: INSTR_STAT_INC(io_exit); /* legacy 设备模拟：COM1 UART / PIC / PIT / 其余端口吞掉 */ handle_io(qualification, rip, instr_len, &resume); break; case EXIT_INTERRUPT_WINDOW: INSTR_STAT_INC(intr_window); /* guest 刚开中断（RFLAGS.IF=1）：尝试注入 pending IRQ */ { u64 cpu = vmx_vmcs_read(VMCS_CPU_BASED_VM_EXEC_CONTROL); vmx_vmcs_write(VMCS_CPU_BASED_VM_EXEC_CONTROL, cpu & ~CPU_BASED_INTR_WINDOW_EXITING); maybe_inject_irq(); resume = 1; } break; case EXIT_VMX_PREEMPTION_TIMER: INSTR_STAT_INC(preempt_timer); /* 10ms 周期 exit：resume 前由 vmexit_before_resume 统一 * 轮询 host 串口 RX / 推进 PIT tick / 注入 pending IRQ。 * Also force IRQ0 here. TSC-based PIT accounting can miss ticks * after park/resume, and nanosleep/Xorg then freeze on noapic. */ vmx_guest_queue_irq(g_pic_vec_base ? g_pic_vec_base : 0x30); resume = 1; if (g_linux_guest_active && g_linux_timeslice_armed) { g_linux_slice_ticks++; if (g_linux_slice_ticks >= LINUX_HOST_SLICE_TICKS) { g_linux_slice_ticks = 0; g_guest_parked = 1; resume = 0; } } break; case EXIT_RDMSR: case EXIT_WRMSR: { INSTR_STAT_INC(msr_exit); /* 真机 MSR 访问比 QEMU 严格得多。Linux guest 会访问许多 MSR： * - IA32_MISC_ENABLE (0x1A0): QEMU 忽略，真机需要返回合理值 * - IA32_APIC_BASE (0x1B): APIC base 地址 * - IA32_TSC_AUX (0xC0000103): RDTSCP auxiliary * - IA32_FS_BASE/GS_BASE (0xC0000100/1): 已由 VMCS 管理 * - IA32_KERNEL_GS_BASE (0xC0000102): swapgs 目标 * - IA32_BNDCFGS (0xD90): MPX bound config（某些 Intel 有） * 当前策略：直接 passthrough host MSR 值（读），或忽略写。 * 注意：WRMSR 写入某些 MSR 可能影响 host，需要白名单保护。 */ u64 msr = g_guest_regs.rcx & 0xFFFFFFFFULL; if (msr == 0xC0000080ULL) { /* IA32_EFER */ if (reason == EXIT_RDMSR) { u64 val = vmx_vmcs_read(VMCS_GUEST_IA32_EFER); g_guest_regs.rax = val & 0xFFFFFFFFULL; g_guest_regs.rdx = (val >> 32) & 0xFFFFFFFFULL; } else { u64 val = (g_guest_regs.rax & 0xFFFFFFFFULL) | (g_guest_regs.rdx << 32); val |= EFER_LME | EFER_LMA; val &= ~EFER_SVME; vmx_vmcs_write(VMCS_GUEST_IA32_EFER, val); } vmx_vmcs_write(VMCS_GUEST_RIP, rip + instr_len); resume = 1; break; } /* FS/GS bases live in the VMCS. Ignoring WRMSR leaves TLS at 0. */ if (msr == 0xC0000100ULL || msr == 0xC0000101ULL || msr == 0xC0000102ULL) { static u64 guest_kernel_gs; if (reason == EXIT_RDMSR) { u64 val = 0; if (msr == 0xC0000100ULL) { val = vmx_vmcs_read(VMCS_GUEST_FS_BASE); } else if (msr == 0xC0000101ULL) { val = vmx_vmcs_read(VMCS_GUEST_GS_BASE); } else { val = guest_kernel_gs; } g_guest_regs.rax = val & 0xFFFFFFFFULL; g_guest_regs.rdx = (val >> 32) & 0xFFFFFFFFULL; } else { u64 val = (g_guest_regs.rax & 0xFFFFFFFFULL) | (g_guest_regs.rdx << 32); if (msr == 0xC0000100ULL) { vmx_vmcs_write(VMCS_GUEST_FS_BASE, val); } else if (msr == 0xC0000101ULL) { vmx_vmcs_write(VMCS_GUEST_GS_BASE, val); } else { u32 lo = (u32)val; u32 hi = (u32)(val >> 32); guest_kernel_gs = val; /* SWAPGS reads the hardware KERNEL_GS_BASE MSR. */ __asm__ volatile("wrmsr" :: "a"(lo), "d"(hi), "c"(0xC0000102u)); } } vmx_vmcs_write(VMCS_GUEST_RIP, rip + instr_len); resume = 1; break; } if (reason == EXIT_RDMSR) { u64 val = 0; /* 白名单：允许读取的 MSR（passthrough host 值） */ int allowed = 0; /* 通用允许列表 */ if (msr == 0x1A0 || /* IA32_MISC_ENABLE */ msr == 0x1B || /* IA32_APIC_BASE */ msr == 0xC0000100 || /* IA32_FS_BASE */ msr == 0xC0000101 || /* IA32_GS_BASE */ msr == 0xC0000102 || /* IA32_KERNEL_GS_BASE */ msr == 0xC0000103 || /* IA32_TSC_AUX */ msr == 0xD90 || /* IA32_BNDCFGS */ msr == 0x3A || /* IA32_FEATURE_CONTROL */ msr == 0x8B || /* IA32_BIOS_SIGN (microcode) */ msr == 0x17 || /* IA32_PLATFORM_ID */ msr == 0x4B || /* IA32_UCODE_REV (some CPUs) */ msr == 0xCE || /* IA32_PLATFORM_INFO */ msr == 0x198 || /* IA32_PERF_STATUS */ msr == 0x199 || /* IA32_PERF_CTL */ msr == 0x1AD || /* IA32_THREAD_FEEDBACK_CHAR (Raptor Lake) */ msr == 0x1AE || /* IA32_CORE_THREAD_COUNT */ (msr >= 0x0 && msr <= 0x1F) || /* Basic CPUID MSRs */ (msr >= 0x30 && msr <= 0x3F) || /* PMC MSRs */ (msr >= 0x186 && msr <= 0x18F) || /* Perf MSRs */ (msr >= 0xC0000000 && msr <= 0xC0000103) /* SYSENTER/EFER etc */ ) { allowed = 1; } if (allowed) { /* 安全读取：用 rdmsr 可能 #GP，用 try-catch 模式 */ __asm__ volatile( "1: rdmsr\n" " jmp 2f\n" ".section .text\n" "2:\n" : "=a"(((u32*)&val)[0]), "=d"(((u32*)&val)[1]) : "c"(msr) ); } g_guest_regs.rax = val & 0xFFFFFFFFULL; g_guest_regs.rdx = (val >> 32) & 0xFFFFFFFFULL; } else { /* WRMSR：白名单保护，防止 guest 修改 host MSR */ int allowed = 0; if (msr == 0xC0000100 || /* IA32_FS_BASE — handled above */ msr == 0xC0000101 || msr == 0xC0000102 || msr == 0xC0000103 || /* IA32_TSC_AUX */ msr == 0x1B || /* IA32_APIC_BASE — 忽略（guest noapic） */ msr == 0xD90 || /* IA32_BNDCFGS — 忽略 */ msr == 0x199 || /* IA32_PERF_CTL — 忽略 */ msr == 0x6A0 || /* IA32_U_CET */ msr == 0x6A2 || /* IA32_S_CET */ msr == 0x6A4 || msr == 0x6A5 || msr == 0x6A6 || msr == 0x6A7 || msr == 0x6A8 ) { allowed = 1; } /* 不允许的写直接吞掉 */ if (!allowed) { log_hex64("[VMEXIT] WRMSR blocked msr=", msr); } } vmx_vmcs_write(VMCS_GUEST_RIP, rip + instr_len); resume = 1; break; } case EXIT_VMCALL: INSTR_PROBE(HCALL, g_guest_regs.rax, g_guest_regs.rdi, g_guest_regs.rsi, rip); /* VMCALL hypercall: read guest GPRs from g_guest_regs (saved by * vmexit_asm.S), dispatch to the hypercall handler, and write * the return value into g_guest_regs.rax. The handler also * advances GUEST_RIP past the VMCALL instruction. */ { int hcall_resume = hypercall_handle( g_guest_regs.rax, g_guest_regs.rdi, g_guest_regs.rsi, g_guest_regs.rdx, rip, instr_len); resume = hcall_resume; } break; default: log_exit_diagnostics(reason, qualification, rip, instr_len); resume = 0; break; } (void)qualification; /* resume 前例行工作：仅在继续跑 guest 时注入 IRQ / 轮询设备。 * 不要在 HLT/timeslice 归还 host 时调用——那会在 host 栈上走 guest * virtio poll，已触发 CR2=3 的 host #PF。下次 linux_resume 入口再准备。 */ if (resume) { INSTR_STAT_INC(pre_resume); vmexit_before_resume(); } return resume; } u64 vmexit_get_count(void) { return g_vmexit_count; }