Deaicup/Deshab-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FUCKYOU

Browse Source
This commit is contained in:
xyou
2026-07-31 13:53:49 +08:00
parent 5ea84fac97
commit f52157b7af
241 changed files with 26843 additions and 8155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CODE/DKM/acpi/acpi.c
+180
View File
@@ -103,6 +103,15 @@ typedef struct __attribute__((packed)) {
u32 creator_revision;
} acpi_sdt_header;
/* ACPI Generic Address Structure (GAS) — HPET 等表使用 */
typedef struct __attribute__((packed)) {
u8 space_id; /* 0=SystemMemory, 1=SystemIO, 2=PCIConfig, ... */
u8 bit_width;
u8 bit_offset;
u8 access_size; /* 0=undefined, 1=byte, 2=word, 3=dword, 4=qword */
u64 address;
} acpi_generic_address;
/* ---------------------------------------------------------------
* Driver descriptor
* --------------------------------------------------------------- */
@@ -212,6 +221,168 @@ static const acpi_rsdp_v2 *acpi_find_rsdp(const void *limine_rsdp) {
return NULL;
}
/* ---------------------------------------------------------------
* MCFG (PCIe ECAM Memory-mapped Configuration) 解析
*
* **QEMU 无法测试但真机必需**: PCIe 配置空间通过 ECAM 访问,
* MCFG 表提供 ECAM 基址和总线范围。没有 MCFG 解析,
* 真机无法访问 Bus 0 以外的 PCIe 设备配置空间。
* QEMU 的 PCI 设备都在 Bus 0, 使用传统 CF8/CFC 端口即可。
* --------------------------------------------------------------- */
typedef struct __attribute__((packed)) {
u64 base_address; /* ECAM 基址 (4K 对齐, 256MB per bus) */
u16 pci_segment_group; /* PCI 段组号 (通常为 0) */
u8 start_bus_number; /* 起始总线号 */
u8 end_bus_number; /* 结束总线号 */
u32 reserved;
} mcfg_entry;
typedef struct __attribute__((packed)) {
acpi_sdt_header hdr;
u64 reserved; /* 保留, 通常为 0 */
/* mcfg_entry entries[] 紧跟 */
} mcfg_table;
static void acpi_parse_mcfg(const acpi_sdt_header *hdr) {
if (!hdr || hdr->length < sizeof(mcfg_table)) return;
const mcfg_table *mcfg = (const mcfg_table *)hdr;
u32 entry_size = sizeof(mcfg_entry);
u32 entries_len = mcfg->hdr.length - sizeof(mcfg_table);
u32 count = entries_len / entry_size;
g_log->info("[acpi] MCFG entries:");
log_hex("[acpi] count=", count);
for (u32 i = 0; i < count; i++) {
const mcfg_entry *e = (const mcfg_entry *)((const u8 *)mcfg + sizeof(mcfg_table) + i * entry_size);
log_hex("[acpi] MCFG base=", e->base_address);
log_hex("[acpi] MCFG seg=", e->pci_segment_group);
log_hex("[acpi] MCFG bus_start=", e->start_bus_number);
log_hex("[acpi] MCFG bus_end=", e->end_bus_number);
/* 验证 ECAM 基址合理性 */
if (e->base_address == 0 || e->base_address == 0xFFFFFFFFFFFFFFFFULL) {
g_log->warn("[acpi] MCFG entry has invalid base address");
continue;
}
/* 真机: ECAM 基址通常在 0xB0000000 - 0xBFFFFFFF (3G-4G 范围),
* 但也可能在 4G 以上 (特别是大内存系统) */
if (e->base_address >= 0x100000000ULL) {
g_log->info("[acpi] MCFG base above 4G — requires MMIO mapping");
}
}
}
/* ---------------------------------------------------------------
* HPET (High Precision Event Timer) 解析
*
* 真机需要 HPET 替代 PIT 作为高精度定时器源。
* QEMU 默认也提供 HPET, 但当前内核使用 PIT 即可。
* HPET 信息留作后续高精度定时器实现的参考。
* --------------------------------------------------------------- */
typedef struct __attribute__((packed)) {
u8 hardware_rev_id;
u8 comparator_count : 5;
u8 counter_size : 1; /* 0=32-bit, 1=64-bit */
u8 reserved0 : 1;
u8 legacy_replacement : 1; /* 1=LegacyReplacement IRQ routing available */
u16 pci_vendor_id;
acpi_generic_address address; /* HPET 寄存器基址 */
u8 hpet_number;
u16 minimum_tick;
u8 page_protection;
} hpet_entry_data;
/* HPET 表不含子条目, 只有 1 个 data block */
typedef struct __attribute__((packed)) {
acpi_sdt_header hdr;
u32 event_timer_block_id; /* 与 hpet_entry_data 的前 4 字节相同 */
acpi_generic_address base_address;
u8 hpet_number;
u16 minimum_tick;
u8 page_protection;
} hpet_table;
static void acpi_parse_hpet(const acpi_sdt_header *hdr) {
if (!hdr || hdr->length < sizeof(hpet_table)) return;
const hpet_table *hpet = (const hpet_table *)hdr;
u32 block_id = hpet->event_timer_block_id;
u8 comp_count = (block_id >> 8) & 0x1f;
u8 count_size = (block_id >> 13) & 1;
u8 legacy = (block_id >> 15) & 1;
u8 rev_id = block_id & 0xff;
u16 vendor_id = (block_id >> 16) & 0xffff;
g_log->info("[acpi] HPET info:");
log_hex("[acpi] rev_id=", rev_id);
log_hex("[acpi] comparators=", comp_count);
log_hex("[acpi] counter_size=", count_size ? 64 : 32);
log_hex("[acpi] legacy_irq=", legacy);
log_hex("[acpi] vendor_id=", vendor_id);
log_hex("[acpi] min_tick=", hpet->minimum_tick);
/* HPET 地址空间类型 */
if (hpet->base_address.space_id == 0) {
/* System Memory — MMIO 访问 */
u64 addr = hpet->base_address.address;
log_hex("[acpi] HPET MMIO addr=", addr);
if (addr >= 0x100000000ULL) {
g_log->info("[acpi] HPET base above 4G — requires MMIO mapping");
}
} else if (hpet->base_address.space_id == 1) {
/* System I/O — 端口 I/O */
log_hex("[acpi] HPET I/O port=", hpet->base_address.address);
}
}
/* ---------------------------------------------------------------
* MADT (APIC) 解析 — 为 apic 驱动提供数据
* --------------------------------------------------------------- */
typedef struct __attribute__((packed)) {
acpi_sdt_header hdr;
u32 lapic_addr;
u32 flags;
} acpi_madt;
static void acpi_parse_madt(const acpi_sdt_header *hdr) {
if (!hdr || hdr->length < sizeof(acpi_madt)) return;
const acpi_madt *madt = (const acpi_madt *)hdr;
g_log->info("[acpi] MADT detail:");
log_hex("[acpi] LAPIC addr=", madt->lapic_addr);
log_hex("[acpi] flags=", madt->flags);
if (madt->flags & 1) {
g_log->info("[acpi] PCAT_COMPAT=1 (8259 PIC present)");
}
/* 枚举 MADT 子条目做汇总统计 */
const u8 *base = (const u8 *)madt;
u32 off = sizeof(acpi_madt);
u32 lapic_count = 0, ioapic_count = 0, iso_count = 0, nmi_count = 0;
while (off + 2 <= madt->hdr.length) {
const u8 *entry = base + off;
u8 type = entry[0];
u8 len = entry[1];
if (len < 2 || off + len > madt->hdr.length) break;
if (type == 0) lapic_count++;
else if (type == 1) ioapic_count++;
else if (type == 2) iso_count++;
else if (type == 3 || type == 4) nmi_count++;
off += len;
}
log_hex("[acpi] LAPIC entries=", lapic_count);
log_hex("[acpi] IOAPIC entries=", ioapic_count);
log_hex("[acpi] ISO entries=", iso_count);
log_hex("[acpi] NMI entries=", nmi_count);
}
/* ---------------------------------------------------------------
* Table enumeration
* --------------------------------------------------------------- */
@@ -241,6 +412,15 @@ static void acpi_enumerate_rsdt(const acpi_sdt_header *rsdt, u32 entry_size) {
g_log->info(sig);
log_hex("[acpi] addr=", addr);
log_hex("[acpi] len =", (u64)hdr->length);
/* 对关键表做详细解析 */
if (sig[0]=='A' && sig[1]=='P' && sig[2]=='I' && sig[3]=='C') {
acpi_parse_madt(hdr);
} else if (sig[0]=='M' && sig[1]=='C' && sig[2]=='F' && sig[3]=='G') {
acpi_parse_mcfg(hdr);
} else if (sig[0]=='H' && sig[1]=='P' && sig[2]=='E' && sig[3]=='T') {
acpi_parse_hpet(hdr);
}
}
}
CODE/DKM/ahci/ahci.c
+102 -20
View File
@@ -5,6 +5,13 @@
*/
#include "../dkm_shared.h"
#include "../dkm_instr.h"
DKM_STAT_DECL(ahci_port_found);
DKM_STAT_DECL(ahci_identify_ok);
DKM_STAT_DECL(ahci_block_reads);
DKM_STAT_DECL(ahci_block_writes);
DKM_TS_DECL(ts_abar);
static const char *const g_depends[] = { "pci", "irq" };
static const char *const g_provides[] = { "block" };
@@ -130,7 +137,8 @@ static void ahci_zero(void *ptr, u32 len) {
}
static int ahci_find(u8 *out_bus, u8 *out_dev, u8 *out_func) {
for (u8 bus = 0; bus < 8; bus++) {
/* PCI 规范允许 256 条 bus (0-255)，实机 AHCI 可能位于 bus > 7 */
for (u16 bus = 0; bus < 256; bus++) {
for (u8 dev = 0; dev < 32; dev++) {
u32 vd = pci_read(bus, dev, 0, PCI_VENDOR_ID);
if ((vd & 0xffff) == 0xffff) continue;
@@ -298,13 +306,16 @@ static int ahci_read_blocks_internal(u32 port, ahci_cmd_header *hdr, struct dkm_
g_abar[(base + PxIS) / 4] = 0xffffffffu;
g_abar[HBA_IS / 4] = (1u << port);
g_abar[(base + PxCI) / 4] = 1;
for (u32 i = 0; i < 5000000; i++) {
/* TSC deadline 超时: 5 秒，实机机械 HDD 首次寻道需数百 ms */
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 5000;
while (dkm_rdtsc() < deadline) {
if ((g_abar[(base + PxCI) / 4] & 1u) == 0) {
u8 *dst = (u8 *)buffer;
u8 *src = (u8 *)data->virt;
for (u32 j = 0; j < bytes; j++) dst[j] = src[j];
return 0;
}
__asm__ volatile("pause");
}
g_log->warn("[ahci] READ timeout");
log_hex("[ahci] READ lba=", lba);
@@ -330,6 +341,10 @@ static int ahci_write_blocks_internal(u32 port, ahci_cmd_header *hdr, struct dkm
for (u32 j = 0; j < bytes; j++) dst[j] = src[j];
}
/* BUG-P0-2: 写入前必须 flush CPU cache，否则 DMA 控制器从物理内存
* 读到陈旧数据（CPU write-back cache 尚未写回）。实机必现。 */
__asm__ volatile("wbinvd" ::: "memory");
hdr[0].flags = 5 | (1u << 6); /* CFL=5, W=1 (write) */
hdr[0].prdtl = 1;
hdr[0].prdbc = 0;
@@ -341,29 +356,39 @@ static int ahci_write_blocks_internal(u32 port, ahci_cmd_header *hdr, struct dkm
tbl->prdt[0].dbc_i = (bytes - 1) | (1u << 31);
ahci_fill_write_fis(tbl->cfis, lba, count);
/* set PxCMD.W to indicate write direction */
u32 cmd = g_abar[(base + PxCMD) / 4];
cmd |= AHCI_CMD_W;
g_abar[(base + PxCMD) / 4] = cmd;
/* P0 修复: 移除 PxCMD.W 写入。AHCI 规范中 PxCMD.W 是只读位，
* 由硬件根据命令头 flags.W 自动设置。软件写入此位违反规范，
* 可能在某些实机 AHCI 控制器上导致未定义行为。 */
/* (原代码: cmd |= AHCI_CMD_W; g_abar[...] = cmd; — 已删除) */
g_abar[(base + PxIS) / 4] = 0xffffffffu;
g_abar[HBA_IS / 4] = (1u << port);
g_abar[(base + PxCI) / 4] = 1;
for (u32 i = 0; i < 5000000; i++) {
/* P0 修复: 写入超时改为 TSC deadline，与读取路径一致。
* 固定循环在实机 HDD 上会因 CPU 频率差异导致提前超时或死循环。 */
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 5000;
while (dkm_rdtsc() < deadline) {
if ((g_abar[(base + PxCI) / 4] & 1u) == 0) {
/* clear PxCMD.W after write completes */
cmd = g_abar[(base + PxCMD) / 4];
cmd &= ~AHCI_CMD_W;
g_abar[(base + PxCMD) / 4] = cmd;
/* BUG-P1-1 fix: CI 清除后检查 TFD 错误位。
* TFD 低字节是 STS，正常值 0x50 (DRDY|DRQ) 非零但不是错误。
* 只有 STS bit0 (ERR) 或 ERR 寄存器 (bits15:8) 非零才是真正错误。
* BUG-20260730-P14: 原 `tfd & 0xFF` 误判 STS=0x50 为错误，导致写入
* 实际成功却被报告为失败，FAT32 写路径提前返回错误码。 */
u32 tfd = g_abar[(base + PxTFD) / 4];
u32 is = g_abar[(base + PxIS) / 4];
u8 sts = (u8)(tfd & 0xFF);
u8 err_reg = (u8)((tfd >> 8) & 0xFF);
if ((sts & 0x01) || err_reg) {
g_log->warn("[ahci] write error: TFD ERR");
log_hex("[ahci] TFD=", tfd);
log_hex("[ahci] PxIS=", is);
return -4;
}
return 0;
}
__asm__ volatile("pause");
}
/* timeout: clear PxCMD.W */
cmd = g_abar[(base + PxCMD) / 4];
cmd &= ~AHCI_CMD_W;
g_abar[(base + PxCMD) / 4] = cmd;
g_log->warn("[ahci] WRITE timeout");
log_hex("[ahci] WRITE lba=", lba);
log_hex("[ahci] WRITE count=", count);
@@ -376,6 +401,7 @@ static int ahci_write_blocks_internal(u32 port, ahci_cmd_header *hdr, struct dkm
static int ahci_block_read(void *ctx, u64 lba, u32 count, void *buffer) {
(void)ctx;
if (g_ready_port == 0xffffffffu || !g_cmd_header) return -1;
DKM_STAT_INC(ahci_block_reads);
log_hex("[ahci] block_read lba=", lba);
log_hex("[ahci] block_read count=", count);
while (count > 0) {
@@ -392,6 +418,7 @@ static int ahci_block_read(void *ctx, u64 lba, u32 count, void *buffer) {
static int ahci_block_write(void *ctx, u64 lba, u32 count, const void *buffer) {
(void)ctx;
if (g_ready_port == 0xffffffffu || !g_cmd_header) return -1;
DKM_STAT_INC(ahci_block_writes);
/* ensure port is in ST+FRE state */
u32 base = HBA_PORT_BASE + g_ready_port * HBA_PORT_SIZE;
u32 cmd = g_abar[(base + PxCMD) / 4];
@@ -445,8 +472,27 @@ static int ahci_identify_port(u32 port, const struct dkm_dma_api *dma) {
struct dkm_dma_buffer data;
if (dma->alloc_pages(1, 1024, 0x100000000ULL, &clb) != 0) return -2;
if (dma->alloc_pages(1, 256, 0x100000000ULL, &fis) != 0) return -3;
if (dma->alloc_pages(1, 128, 0x100000000ULL, &table) != 0) return -4;
if (dma->alloc_pages(1, 512, 0x100000000ULL, &data) != 0) return -5;
/* BUG-P0-1: sizeof(ahci_cmd_table)=144, 128B 缓冲区越界 16 字节。
* 改为 256B 满足对齐和大小要求，防止 ahci_zero 越界写。 */
if (dma->alloc_pages(1, 256, 0x100000000ULL, &table) != 0) return -4;
/* BUG-20260730-022: DMA data buffer must hold at least 8 sectors (4096B)
* for ahci_block_write chunked DMA. 512B caused 3584B out-of-bounds
* overwrite of adjacent kernel data on real hardware. */
if (dma->alloc_pages(1, 8192, 0x100000000ULL, &data) != 0) return -5;
/* P0 验证: DMA 缓冲区物理地址必须在 4GB 以下。
* AHCI 控制器 8086:2922 只支持 32 位 DMA，
* PRDT 地址超过 4GB 会导致 DMA 写入截断到错误物理地址。
* dma->alloc_pages 的上限参数 0x100000000ULL 已保证此约束，
* 但添加显式校验以防御 alloc_pages 实现变更。 */
if (clb.phys >= 0x100000000ULL || fis.phys >= 0x100000000ULL ||
table.phys >= 0x100000000ULL || data.phys >= 0x100000000ULL) {
g_log->warn("[ahci] DMA buffer above 4GB — 32-bit DMA will fail on this controller!");
log_hex("[ahci] clb.phys=", clb.phys);
log_hex("[ahci] fis.phys=", fis.phys);
log_hex("[ahci] table.phys=", table.phys);
log_hex("[ahci] data.phys=", data.phys);
}
if (ahci_stop_port(port) != 0) {
g_log->warn("[ahci] port stop timeout");
@@ -476,7 +522,10 @@ static int ahci_identify_port(u32 port, const struct dkm_dma_api *dma) {
ahci_start_port(port);
g_abar[(base + PxCI) / 4] = 1;
for (u32 i = 0; i < 5000000; i++) {
/* BUG-P0-3: 固定 500 万次循环在 3GHz CPU 上仅 ~1-2ms，
* 不够 HDD 寻道。改为 TSC deadline 5s 超时，与 read/write 路径一致。 */
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 5000;
while (dkm_rdtsc() < deadline) {
if ((g_abar[(base + PxCI) / 4] & 1u) == 0) {
log_hex("[ahci] IDENTIFY port=", port);
log_hex("[ahci] IDENTIFY status=", g_abar[(base + PxTFD) / 4]);
@@ -492,6 +541,7 @@ static int ahci_identify_port(u32 port, const struct dkm_dma_api *dma) {
g_data = data;
return 1;
}
__asm__ volatile("pause");
}
g_log->warn("[ahci] IDENTIFY timeout");
@@ -510,6 +560,7 @@ int driver_init(const struct dkm_kernel_api *api,
if (!api || !api->log) return -1;
g_log = api->log;
dkm_instr_init(api);
g_log->info("[ahci] init begin");
@@ -536,7 +587,17 @@ int driver_init(const struct dkm_kernel_api *api,
pci_write(bus, dev, func, PCI_COMMAND, command);
log_hex("[ahci] PCI command=", command);
u64 abar_phys = (u64)(pci_read(bus, dev, func, PCI_BAR5) & 0xFFFFFFF0u);
/* 读取 ABAR: BAR5 可能是 64-bit MMIO BAR，高位在 BAR4 */
u32 bar5_lo = pci_read(bus, dev, func, PCI_BAR5);
u64 abar_phys;
if ((bar5_lo & 0x7) == 0x4) {
/* 64-bit MMIO BAR: 高 32 位在 BAR4 */
u32 bar5_hi = pci_read(bus, dev, func, PCI_BAR4);
abar_phys = ((u64)bar5_hi << 32) | (bar5_lo & 0xFFFFFFF0u);
g_log->info("[ahci] ABAR is 64-bit MMIO BAR");
} else {
abar_phys = bar5_lo & 0xFFFFFFF0u;
}
u32 irq_line = pci_read(bus, dev, func, PCI_IRQ_LINE) & 0xff;
log_hex("[ahci] ABAR phys=", abar_phys);
log_hex("[ahci] PCI IRQ line=", irq_line);
@@ -548,6 +609,7 @@ int driver_init(const struct dkm_kernel_api *api,
}
g_abar = (volatile u32 *)(uintptr_t)(api->hhdm_offset + abar_phys);
DKM_TS_BEGIN(ts_abar);
log_hex("[ahci] HHDM offset=", api->hhdm_offset);
u32 cap = g_abar[HBA_CAP / 4];
@@ -565,13 +627,32 @@ int driver_init(const struct dkm_kernel_api *api,
log_hex("[ahci] VS=", vs);
log_hex("[ahci] CAP2=", cap2);
log_hex("[ahci] BOHC=", bohc);
/* AHCI 1.3+ BOHC handoff: 如果 BIOS 仍拥有控制器, 请求所有权 */
if (bohc & 0x01) {
g_log->info("[ahci] BIOS owns HBA, requesting handoff");
g_abar[HBA_BOHC / 4] = bohc | 0x02; /* set OOS */
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 2000;
while (dkm_rdtsc() < deadline) {
if (!(g_abar[HBA_BOHC / 4] & 0x01)) break;
__asm__ volatile("pause");
}
if (g_abar[HBA_BOHC / 4] & 0x01) {
g_log->warn("[ahci] BIOS handoff timeout, proceeding anyway");
} else {
g_log->info("[ahci] BIOS handoff complete");
}
}
log_hex("[ahci] ports count=", (cap & 0x1f) + 1);
log_hex("[ahci] cmd slots=", ((cap >> 8) & 0x1f) + 1);
DKM_TS_END(ts_abar, g_log->info, "ahci_abar_mmio");
u32 implemented = 0;
for (u32 port = 0; port < 32; port++) {
if (pi & (1u << port)) {
implemented++;
DKM_STAT_INC(ahci_port_found);
ahci_log_port(port);
}
}
@@ -584,6 +665,7 @@ int driver_init(const struct dkm_kernel_api *api,
int ident = ahci_identify_port(port, api->dma);
if (ident > 0) {
identified++;
DKM_STAT_INC(ahci_identify_ok);
break;
}
}
CODE/DKM/apic/apic.c
+294 -6
View File
@@ -1,7 +1,17 @@
/* DKM APIC Driver — APIC discovery with PIC fallback
/* DKM APIC Driver — APIC discovery + IOAPIC init + LAPIC enable
* Stage 0, required, depends on "timer", provides "irq" and "apic".
* Current kernel IRQ backend is still PIC; this driver only discovers APIC
* topology and keeps PIC routing active.
*
* 真机路径:
* 1. LAPIC 使能 + Spurious Interrupt Vector 注册
* 2. IOAPIC 重定向表初始化: 逐条目编程, 将 ISA IRQ 映射到 vector 0x20-0x2F
* (兼容 PIC 布局), 逐条目 mask
* 3. MADT ISO (Interrupt Source Override) 处理: 按 ACPI 规范修正 IRQ→GSI 映射
* 4. SIPI 多核启动框架 (预留, 未实现)
*
* QEMU 兼容:
* - QEMU 默认只提供 1 个 IOAPIC, 24 条 GSI
* - 实机可能有多个 IOAPIC, GSI 范围更大
* - 实机必须处理 MADT ISO (IRQ0→GSI2 等)
*/
#include <stdint.h>
@@ -47,6 +57,10 @@ struct dkm_kernel_api {
const void *boot_modules_response;
int (*irq_register)(u8 irq, void *handler);
u64 hhdm_offset;
const void *block;
const void *mmio;
void *(*mm_map_mmio)(u64 phys, u64 size);
void (*mm_unmap_mmio)(void *virt, u64 size);
};
struct dkm_driver_handle;
@@ -135,16 +149,49 @@ typedef struct __attribute__((packed)) {
u32 flags;
} acpi_madt;
/* ---- IOAPIC 信息存储（真机多 IOAPIC 支持） ---- */
#define MAX_IOAPIC 4
#define MAX_ISO 16
#define MAX_LAPIC 256
struct ioapic_info {
u8 id;
u32 gsi_base;
u32 gsi_count; /* 从 IOAPIC VER 寄存器读取: max_redir_entry + 1 */
u64 mmio_phys; /* IOAPIC MMIO 物理基址 */
volatile u32 *mmio; /* HHDM/MMIO 窗口映射后的虚拟地址, 0=未映射 */
};
struct iso_info {
u8 bus_source; /* ISA bus source IRQ (0-15) */
u32 gsi; /* 全局系统中断号 */
u16 flags; /* 极性/触发模式标志 */
};
struct lapic_entry {
u8 apic_id;
u32 flags; /* MADT LAPIC flags: bit0=enabled */
};
struct madt_info {
u64 lapic_phys;
u32 flags;
u32 lapic_count;
u32 ioapic_count;
u32 iso_count;
struct ioapic_info ioapics[MAX_IOAPIC];
struct iso_info isos[MAX_ISO];
struct lapic_entry lapics[MAX_LAPIC];
};
static const struct dkm_log_api *g_log;
static __inline__ void apic_mb(void) {
__asm__ volatile("" ::: "memory");
}
static int memeq(const char *a, const char *b, u32 n) {
for (u32 i = 0; i < n; i++) {
if (a[i] != b[i]) return 0;
@@ -255,19 +302,36 @@ static void parse_madt_entries(const acpi_madt *madt, struct madt_info *info) {
}
if (type == 0 && len >= 8) {
/* LAPIC entry */
u8 apic_id = entry[3];
u32 flags = *(const u32 *)(const void *)(entry + 4);
if (flags & 1u) info->lapic_count++;
log_hex("[apic] CPU LAPIC id=", apic_id);
if (flags & 1u) {
info->lapic_count++;
if (info->lapic_count <= MAX_LAPIC) {
info->lapics[info->lapic_count - 1].apic_id = apic_id;
info->lapics[info->lapic_count - 1].flags = flags;
}
}
} else if (type == 1 && len >= 12) {
/* IOAPIC entry */
u8 ioapic_id = entry[2];
u32 addr = *(const u32 *)(const void *)(entry + 4);
u32 gsi = *(const u32 *)(const void *)(entry + 8);
info->ioapic_count++;
log_hex("[apic] IOAPIC id=", ioapic_id);
log_hex("[apic] IOAPIC addr=", addr);
log_hex("[apic] IOAPIC gsi_base=", gsi);
if (info->ioapic_count < MAX_IOAPIC) {
struct ioapic_info *io = &info->ioapics[info->ioapic_count];
io->id = ioapic_id;
io->gsi_base = gsi;
io->mmio_phys = (u64)addr;
io->mmio = (volatile u32 *)(uintptr_t)(u64)addr; /* HHDM 直映射, 后续映射修正 */
io->gsi_count = 0;
info->ioapic_count++;
}
} else if (type == 2 && len >= 10) {
/* ISO (Interrupt Source Override) entry */
u8 source = entry[3];
u32 gsi = *(const u32 *)(const void *)(entry + 4);
u16 flags = *(const u16 *)(const void *)(entry + 8);
@@ -275,6 +339,12 @@ static void parse_madt_entries(const acpi_madt *madt, struct madt_info *info) {
log_hex("[apic] ISO source=", source);
log_hex("[apic] ISO gsi=", gsi);
log_hex("[apic] ISO flags=", flags);
if (info->iso_count <= MAX_ISO) {
struct iso_info *iso = &info->isos[info->iso_count - 1];
iso->bus_source = source;
iso->gsi = gsi;
iso->flags = flags;
}
} else if (type == 5 && len >= 12) {
u64 override = *(const u64 *)(const void *)(entry + 4);
info->lapic_phys = override;
@@ -285,6 +355,210 @@ static void parse_madt_entries(const acpi_madt *madt, struct madt_info *info) {
}
}
/* ---- IOAPIC MMIO 访问 ---- *
* IOAPIC 寄存器通过 MMIO 访问, 地址空间为 IOREGSEL(0x00) + IOWIN(0x10).
* 真机 IOAPIC 地址通常在 0xFEC00000, HHDM 通常覆盖此低 4G 地址。
* 写入 IOREGSEL 选择寄存器号, 读/写 IOWIN 访问数据。
*/
#define IOAPIC_IOREGSEL 0x00
#define IOAPIC_IOWIN 0x10
static u32 ioapic_read(volatile u32 *base, u8 reg) {
base[IOAPIC_IOREGSEL / 4] = (u32)reg;
apic_mb(); /* 寄存器选择必须先于数据读取 */
return base[IOAPIC_IOWIN / 4];
}
static void ioapic_write(volatile u32 *base, u8 reg, u32 val) {
base[IOAPIC_IOREGSEL / 4] = (u32)reg;
apic_mb(); /* 寄存器选择必须先于数据写入 */
base[IOAPIC_IOWIN / 4] = val;
}
/* IOAPIC 重定向表项 (64-bit, 分两次 32-bit 读写)
* bits 0-7: interrupt vector (0x20-0xFF)
* bit 8: delivery mode: 0=fixed, 1=lowest, 2=SMI, 4=NMI, 5=INIT, 7=ExtINT
* bit 9: destination mode: 0=physical, 1=logical
* bit 10: delivery status (RO)
* bit 11: polarity: 0=active high, 1=active low
* bit 12: remote IRR (RO, level-triggered only)
* bit 13: trigger mode: 0=edge, 1=level
* bit 14: mask: 0=enabled, 1=masked
* bit 15: reserved
* bits 16-55: reserved
* bits 56-63: destination APIC ID (physical mode) or MDA (logical mode)
*/
static u64 ioapic_read_redir(volatile u32 *base, u8 entry) {
u32 lo = ioapic_read(base, (u8)(0x10 + entry * 2));
u32 hi = ioapic_read(base, (u8)(0x10 + entry * 2 + 1));
return ((u64)hi << 32) | (u64)lo;
}
static void ioapic_write_redir(volatile u32 *base, u8 entry, u64 val) {
ioapic_write(base, (u8)(0x10 + entry * 2), (u32)(val & 0xffffffffu));
ioapic_write(base, (u8)(0x10 + entry * 2 + 1), (u32)(val >> 32));
}
/* ---- IOAPIC 初始化 ---- *
* 逐条目编程重定向表: ISA IRQ→vector 映射 + mask all。
* 真机关键: 处理 MADT ISO (IRQ0→GSI2 是最常见的 override)。
* PIC 兼容模式: vector = 0x20 + IRQ (与 PIC remap 布局一致)。
*
* **QEMU 无法测试但真机必需的代码路径**:
* 1. ISO 处理: 真机几乎总有 IRQ0→GSI2 override, QEMU 默认无 override
* 2. 极性标志: 真机 ISA IRQ0 (timer) 必须设 active-high,
* 但 ISO 可能指定 active-low (PCIe SCI 等)
* 3. 多 IOAPIC: 真机可能有 2+ IOAPIC, 每个 GSI 范围不同
*/
static void ioapic_init_one(struct ioapic_info *io, u64 hhdm_offset,
const struct iso_info *isos, u32 iso_count,
void *(*mm_map_mmio_fn)(u64, u64)) {
/* 映射 IOAPIC MMIO — 真机地址通常在 0xFECxxxxx, HHDM 通常覆盖,
* 但使用 mm_map_mmio 更安全 (PCD|PWT 属性) */
if (mm_map_mmio_fn) {
void *mapped = mm_map_mmio_fn(io->mmio_phys, 0x1000);
if (mapped) {
io->mmio = (volatile u32 *)mapped;
g_log->info("[apic] IOAPIC mapped via mm_map_mmio");
} else {
g_log->warn("[apic] mm_map_mmio failed for IOAPIC; trying HHDM");
io->mmio = (volatile u32 *)(uintptr_t)(hhdm_offset + io->mmio_phys);
}
} else {
io->mmio = (volatile u32 *)(uintptr_t)(hhdm_offset + io->mmio_phys);
}
/* 读取 IOAPIC 版本寄存器获取最大重定向条目数 */
u32 ver = ioapic_read(io->mmio, 0x01);
u32 max_redir = ((ver >> 16) & 0xff) + 1;
io->gsi_count = max_redir;
log_hex("[apic] IOAPIC VER=", ver);
log_hex("[apic] IOAPIC max_redir=", max_redir);
/* 读取 IOAPIC ID 验证 */
u32 id_reg = ioapic_read(io->mmio, 0x00);
log_hex("[apic] IOAPIC ID reg=", id_reg);
/* 构建默认 ISA IRQ→GSI→vector 映射表 (identity map + ISO override)
* 默认: GSI = ISA_IRQ (0-15), vector = 0x20 + ISA_IRQ
* ISO override: 修改特定 IRQ 的 GSI 和极性/触发模式 */
u32 gsi_to_vector[256]; /* GSI → vector 映射 */
u32 gsi_flags[256]; /* GSI → polar/trigger flags from ISO */
for (u32 i = 0; i < 256; i++) {
gsi_to_vector[i] = 0;
gsi_flags[i] = 0;
}
/* 先建立 identity 映射: GSI 0-15 → vector 0x20-0x2F */
for (u32 gsi = 0; gsi < 16; gsi++) {
gsi_to_vector[gsi] = 0x20 + gsi;
}
/* 应用 ISO override */
for (u32 i = 0; i < iso_count; i++) {
const struct iso_info *iso = &isos[i];
u8 src = iso->bus_source;
u32 gsi = iso->gsi;
if (gsi < 256) {
gsi_to_vector[gsi] = 0x20 + src; /* vector 仍基于 ISA IRQ 编号 */
gsi_flags[gsi] = iso->flags;
g_log->info("[apic] ISO: ISA IRQ -> GSI override");
log_hex("[apic] ISA IRQ=", src);
log_hex("[apic] GSI=", gsi);
log_hex("[apic] vector=", 0x20 + src);
}
}
/* 编程重定向表: 全部 mask, 编入正确的 vector/极性/触发模式/目标 APIC */
for (u32 pin = 0; pin < max_redir && pin < 256; pin++) {
u32 gsi = io->gsi_base + pin;
u64 entry = 0;
if (gsi < 256 && gsi_to_vector[gsi] != 0) {
u32 vec = gsi_to_vector[gsi];
u16 flags = gsi_flags[gsi];
entry = (u64)vec; /* vector */
/* delivery mode = fixed (0) */
/* destination mode = physical (0) */
/* 极性: bit 11. ISA 默认 active-high (0).
* ISO flags bit 1: 0=conforms (active-high for ISA), 1=active-low */
if (flags & 0x03) {
if (flags & 0x02) entry |= (1ULL << 11); /* active low */
}
/* 触发模式: bit 13. ISA 默认 edge (0).
* ISO flags bit 3: 0=conforms (edge for ISA), 1=level */
if (flags & 0x08) {
if (flags & 0x04) entry |= (1ULL << 13); /* level triggered */
}
/* destination: BSP APIC ID (读自 LAPIC) */
/* 暂时硬编码 destination = 0 (BSP), 后续从 LAPIC ID 寄存器获取 */
entry |= (0ULL << 56); /* destination APIC ID = 0 */
}
entry |= (1ULL << 14); /* mask = 1, 所有条目默认屏蔽 */
ioapic_write_redir(io->mmio, (u8)pin, entry);
}
g_log->info("[apic] IOAPIC redirect table programmed (all masked)");
}
/* ---- LAPIC 使能 ---- *
* 真机必须:
* 1. 写 IA32_APIC_BASE MSR 启用 bit 11 (global enable)
* 2. 写 LAPIC SVR 寄存器设置 Spurious Interrupt Vector (建议 0xFF)
* 3. 设置 SVR bit 8 (APIC software enable)
*
* **QEMU 无法测试但真机必需**: LAPIC SVR 设置 — QEMU 默认已启用 LAPIC,
* 但实机 BIOS 可能留 LAPIC 为 software-disable 状态。
*/
static void wrmsr(u32 msr, u64 val) {
u32 lo = (u32)(val & 0xffffffffu);
u32 hi = (u32)(val >> 32);
__asm__ volatile("wrmsr" :: "a"(lo), "d"(hi), "c"(msr));
}
static void lapic_enable(u64 hhdm_offset, u64 lapic_phys) {
volatile u32 *lapic = (volatile u32 *)(uintptr_t)(hhdm_offset + lapic_phys);
/* 确保 IA32_APIC_BASE 的 global enable 和 BSP 位已设置 */
u64 apic_base = rdmsr(0x1B);
apic_base |= (1ULL << 11); /* global enable */
/* 保持 BSP 位不变 (bit 8) */
wrmsr(0x1B, apic_base);
g_log->info("[apic] LAPIC global enable set in MSR");
/* 读当前 SVR */
u32 svr = lapic[0xF0 / 4];
log_hex("[apic] LAPIC SVR before=", svr);
/* 设置 Spurious Interrupt Vector = 0xFF + APIC software enable (bit 8) */
svr = 0xFF | (1u << 8);
lapic[0xF0 / 4] = svr;
apic_mb();
/* 回读验证 */
svr = lapic[0xF0 / 4];
log_hex("[apic] LAPIC SVR after=", svr);
if (svr & (1u << 8)) {
g_log->info("[apic] LAPIC software enabled");
} else {
g_log->warn("[apic] LAPIC SVR bit8 not set — hardware may not support");
}
/* 读 LAPIC ID 和 version 做诊断 dump */
log_hex("[apic] LAPIC ID=", lapic[0x20 / 4]);
log_hex("[apic] LAPIC LDR=", lapic[0xD0 / 4]);
log_hex("[apic] LAPIC version=", lapic[0x30 / 4]);
}
__attribute__((visibility("default")))
int driver_init(const struct dkm_kernel_api *api,
struct dkm_driver_handle *handle) {
@@ -346,7 +620,21 @@ int driver_init(const struct dkm_kernel_api *api,
g_log->warn("[apic] HHDM unavailable; skip LAPIC MMIO read");
}
g_log->info("[apic] discovery complete; PIC routing retained");
/* ---- LAPIC 使能（真机必需，QEMU 也可安全执行） ---- */
if (api->hhdm_offset && info.lapic_phys) {
lapic_enable(api->hhdm_offset, info.lapic_phys);
}
/* ---- IOAPIC 初始化（真机必需，QEMU 也可安全执行） ---- */
for (u32 i = 0; i < info.ioapic_count; i++) {
g_log->info("[apic] initializing IOAPIC");
log_hex("[apic] index=", i);
ioapic_init_one(&info.ioapics[i], api->hhdm_offset,
info.isos, info.iso_count,
api->mm_map_mmio);
}
g_log->info("[apic] discovery + init complete; PIC routing retained as fallback");
g_log->info("[apic] driver ready");
return 0;
}
CODE/DKM/dkm_instr.h
+87
View File
@@ -0,0 +1,87 @@
/* dkm_instr.h — DKM 驱动专用插桩宏（无 UTSM 头文件依赖）
*
* 设计原则:
* - 使用 dkm_shared.h 已有的 dkm_rdtsc()/dkm_tsc_per_ms
* - 日志通过 dkm_kernel_api->log 输出
* - 统计计数器是 file-local static
* - 禁用时全部宏展开为空（零开销）
* - 需在 driver_init 开头调用 dkm_instr_init(api) 初始化
*
* 使用方式:
* 1. #include "../dkm_instr.h"
* 2. 在 driver_init 开头: dkm_instr_init(api);
* 3. 使用 DKM_STAT_INC / DKM_TS / DKM_LOG_TRACE 宏
*/
#ifndef DKM_INSTR_H
#define DKM_INSTR_H
#include "dkm_shared.h"
/* ---- 全局开关（dkm_instr.c 定义，每个 .drv 引用同一符号） ----
* 注意: DKM .drv 是 ET_REL 可重定位对象，每个驱动有独立的数据段副本。
* 因此 dkm_instr_global_enable 在每个 .drv 中独立控制，由
* dkm_instr_init() 在 driver_init 中根据 kernel_api 设置。
*/
static u32 dkm_instr_global_enable __attribute__((unused)) = 0;
/* ---- DKM_STAT: 轻量级计数器 ----
* DKM_STAT_DECL(name) — 在函数/文件作用域声明计数器
* DKM_STAT_INC(name) — 自增（仅当 enabled）
* DKM_STAT_DUMP(name, log_info_fn) — 输出当前计数值
*/
#define DKM_STAT_DECL(name) static u64 name##_count = 0
#define DKM_STAT_INC(name) do { \
if (dkm_instr_global_enable) { name##_count++; } \
} while(0)
#define DKM_STAT_DUMP(name, log_info_fn) do { \
if (dkm_instr_global_enable && name##_count > 0) { \
log_info_fn("[DKM:STAT] " #name " count="); \
/* 简单 u64 to hex 串口输出 */ \
static const char _hx[] = "0123456789abcdef"; \
char _buf[19]; _buf[0]='0'; _buf[1]='x'; \
for(int _i=0;_i<16;_i++) _buf[2+_i]=_hx[(name##_count>>((15-_i)*4))&0xf]; \
_buf[18]=0; log_info_fn(_buf); \
} \
} while(0)
/* ---- DKM_TS: 时间戳测量 ----
* DKM_TS_DECL(name) — 声明时间戳变量
* DKM_TS_BEGIN(name) — 记录起始 TSC
* DKM_TS_END(name, log_info_fn, label) — 计算耗时并输出
*/
#define DKM_TS_DECL(name) static u64 name##_ts = 0
#define DKM_TS_BEGIN(name) do { name##_ts = dkm_rdtsc(); } while(0)
#define DKM_TS_END(name, log_info_fn, label) do { \
if (dkm_instr_global_enable && dkm_tsc_per_ms) { \
u64 _d = dkm_rdtsc() - name##_ts; \
u64 _ms = _d / dkm_tsc_per_ms; \
log_info_fn("[DKM:TS] " label " elapsed_ms="); \
static const char _hx[] = "0123456789abcdef"; \
char _buf[19]; _buf[0]='0'; _buf[1]='x'; \
for(int _i=0;_i<16;_i++) _buf[2+_i]=_hx[(_ms>>((15-_i)*4))&0xf]; \
_buf[18]=0; log_info_fn(_buf); \
} \
} while(0)
/* ---- DKM_LOG_TRACE: 调试级别日志 ----
* 仅当插桩启用时通过 api->log->info 输出
*/
#define DKM_LOG_TRACE(log_api, msg) do { \
if (dkm_instr_global_enable && (log_api)) (log_api)->info(msg); \
} while(0)
/* ---- 初始化：读取 kernel_api 中的插桩标志 ----
* kernel_api->feature_bits bit0 = 插桩启用标志
* 由 UTSM dkm_fill_platform_info() 根据 FUCK debug.dkm_trace 设置
*/
static __inline__ void dkm_instr_init(const struct dkm_kernel_api *api) {
if (api && (api->feature_bits & 0x01)) {
dkm_instr_global_enable = 1;
}
}
#endif /* DKM_INSTR_H */
CODE/DKM/dkm_shared.h
+25 -1
View File
@@ -109,6 +109,15 @@ struct dkm_net_device_desc {
int (*is_wireless)(void *ctx);
};
struct dkm_net_stats {
u64 tx_ok;
u64 tx_err;
u64 rx_ok;
u64 rx_err;
u64 rx_overflow;
u64 link_changes;
};
struct dkm_net_api {
int (*register_device)(const struct dkm_net_device_desc *desc);
u32 (*device_count)(void);
@@ -119,6 +128,7 @@ struct dkm_net_api {
int (*scan_count)(u32 index);
int (*scan_result)(u32 index, u32 n, struct dkm_net_scan_result *out);
int (*is_wireless)(u32 index);
int (*device_stats)(u32 index, struct dkm_net_stats *out);
};
typedef int (*dkm_block_read_fn)(void *ctx, u64 lba, u32 count, void *buffer);
@@ -143,6 +153,16 @@ struct dkm_block_api {
int (*set_write_fn)(u32 index, dkm_block_write_fn fn);
};
/* 与 UTSM/include/utsm/paging.h 中的 dkm_mmio_api 布局一致 */
struct dkm_mmio_api {
/* 四级页表 walk。返回 0=未映射；1=4K PTE；2=2M PDE；3=1G PDPTE */
int (*is_mapped)(u64 vaddr);
/* 把物理区间映射到 hhdm_offset + phys（4KiB UC 页），返回 0 成功 */
int (*map_mmio)(u64 phys, u64 size);
/* 当前 CR3 物理地址（诊断用） */
u64 (*cr3)(void);
};
/* 与 UTSM/include/utsm/dkm.h 中的 dkm_kernel_api 布局完全一致.
* 字段顺序/类型不可调整 — UTSM 端按此偏移填充. */
struct dkm_kernel_api {
@@ -168,7 +188,11 @@ struct dkm_kernel_api {
const void *boot_modules_response;
int (*irq_register)(u8 irq, void *handler);
u64 hhdm_offset;
const struct dkm_block_api *block; /* 末尾字段, 不使用 block 的驱动可不访问 */
const struct dkm_block_api *block; /* 不使用 block 的驱动可不访问 */
const struct dkm_mmio_api *mmio; /* 页表/高位 MMIO 映射服务（HHDM 空洞原地补映射） */
/* —— 新增字段只允许追加在末尾（DSK 以 api+0xA8 硬偏移读取 block） —— */
void *(*mm_map_mmio)(u64 phys, u64 size); /* MMIO 独立窗口映射（PCD|PWT），返回虚拟地址，失败返回 0 */
void (*mm_unmap_mmio)(void *virt, u64 size); /* 解除 MMIO 独立窗口映射 */
};
struct dkm_driver_handle;
CODE/DKM/e1000/e1000.c
+445 -79
View File
@@ -1,9 +1,16 @@
/* DKM e1000 Driver — Intel 8254x Gigabit Ethernet
/* DKM e1000 Driver — Intel 8254x/8257x Gigabit Ethernet
* Stage 3, optional, depends on "pci" and "irq", provides "netdev".
* Finds e1000 via PCI, maps MMIO, reads MAC address.
* Supports classic e1000 (82540EM/82545/82546) and e1000e (82574L/I217/I218/I219).
* Finds device via PCI, maps MMIO, reads MAC (RAL/RAH or EEPROM),
* initializes RX/TX rings, provides link/speed/duplex diagnostics.
*/
#include "../dkm_shared.h"
#include "../dkm_instr.h"
DKM_STAT_DECL(e1k_irq_count);
DKM_STAT_DECL(e1k_rx_packets);
DKM_STAT_DECL(e1k_tx_packets);
static const char *const g_depends[] = {"pci","irq"};
static const char *const g_provides[] = {"netdev"};
@@ -11,7 +18,7 @@ __attribute__((visibility("default"), used))
const struct dkm_driver_desc driver_desc = {
.magic=DKM_DRIVER_MAGIC,.abi_version=DKM_ABI_VERSION,
.desc_size=sizeof(struct dkm_driver_desc),
.name="e1000",.version="0.1.0",.vendor="Deshab",
.name="e1000",.version="0.2.0",.vendor="Deshab",
.driver_class=9,.stage=3,.flags=0,.priority=0,
.depends=g_depends,.depends_count=2,
.provides=g_provides,.provides_count=1,.min_kernel_abi=1,
@@ -41,13 +48,33 @@ const struct dkm_driver_desc driver_desc = {
#define E1000_TXDCTL 0x3828
#define E1000_TCTL 0x0400
#define E1000_TIPG 0x0410
#define E1000_MANC 0x0580
/* STATUS register bits (link/speed/duplex diagnostic) */
#define E1000_STATUS_FD (1u << 0) /* full duplex */
#define E1000_STATUS_LU (1u << 1) /* link up */
#define E1000_STATUS_FUNC_SHIFT 2 /* PCI function number (ro) */
#define E1000_STATUS_TXOFF (1u << 4) /* transmitter paused */
#define E1000_STATUS_SPEEDMASK (3u << 6)
#define E1000_STATUS_SPEED_10 (0u << 6)
#define E1000_STATUS_SPEED_100 (1u << 6)
#define E1000_STATUS_SPEED_1000 (2u << 6) /* classic e1000 only */
/* CTRL register bits */
#define E1000_CTRL_FD (1u << 0)
#define E1000_CTRL_SLU (1u << 6)
#define E1000_CTRL_RST (1u << 26) /* software reset */
#define E1000_CTRL_ASDE (1u << 11) /* auto-speed detect enable */
#define E1000_CTRL_FRCSPD (1u << 12) /* force speed */
#define E1000_CTRL_FRCDPLX (1u << 13) /* force duplex */
/* e1000e (82574L/I217/I219) speed encoding in STATUS differs */
#define E1000_STATUS_SPEED_1000_E1000E (3u << 6)
#define E1000_RX_DESC_COUNT 8u
#define E1000_TX_DESC_COUNT 8u
#define E1000_RX_BUF_SIZE 2048u
#define E1000_TX_BUF_SIZE 2048u
#define E1000_POOL_BYTES(count, size) ((u32)((count) * (size)))
#define E1000_POOL_PAGES(bytes) (((bytes) + 4095u) / 4096u)
#define E1000_RCTL_EN (1u << 1)
#define E1000_RCTL_SBP (1u << 2)
@@ -67,8 +94,97 @@ const struct dkm_driver_desc driver_desc = {
#define E1000_TX_CMD_RS (1u << 3)
#define E1000_TX_STA_DD (1u << 0)
#define E1000_RX_STA_DD (1u << 0)
#define E1000_CTRL_FD (1u << 0)
#define E1000_CTRL_SLU (1u << 6)
/* ---- e1000 device ID table ---- */
/* 分类: 0=经典 e1000 (8254x), 1=e1000e (8257x/I217/I218/I219) */
struct e1000_device_id {
u16 did;
u8 family; /* 0=classic, 1=e1000e */
const char *name;
};
static const struct e1000_device_id g_e1000_ids[] = {
/* 经典 e1000 (8254x 系列) — EERD 接口兼容 */
{0x100E, 0, "82540EM"}, /* QEMU e1000 default */
{0x1004, 0, "82543GC"}, /* Copper */
{0x100F, 0, "82545EM"}, /* Copper */
{0x1010, 0, "82546EB"}, /* Copper dual-port */
{0x1011, 0, "82545EM-Fiber"},
{0x1012, 0, "82546EB-Fiber"},
{0x1013, 0, "82541EI"},
{0x1015, 0, "82540EM-LOM"},
{0x1016, 0, "82540EP"},
{0x1017, 0, "82540EP-LOM"},
{0x1018, 0, "82541EI-LOM"},
{0x1019, 0, "82547EI"},
{0x101A, 0, "82547EI-LOM"},
{0x101D, 0, "82546EB-Quad"},
{0x1026, 0, "82545GM"},
{0x1027, 0, "82545GM-Fiber"},
{0x1028, 0, "82545GM-SerDes"},
{0x1075, 0, "82547GI"},
{0x1076, 0, "82541GI"},
{0x1077, 0, "82541GI-LOM"},
{0x1078, 0, "82541ER"},
{0x1079, 0, "82546GB"},
{0x107A, 0, "82546GB-Fiber"},
{0x107B, 0, "82546GB-SerDes"},
{0x107C, 0, "82541GI-Mobile"},
{0x108B, 0, "82573E"},
{0x108C, 0, "82573E-IAMT"},
{0x109A, 0, "82573L"},
/* e1000e (8257x/I217/I218/I219) 系列 — 寄存器兼容但 EERD/STATUS 编码有差异 */
{0x10D3, 1, "82574L"}, /* QEMU e1000e default */
{0x10F5, 1, "82578DM"},
{0x10F6, 1, "82578DC"},
{0x1502, 1, "82579V"},
{0x1503, 1, "82579V-2"},
{0x1533, 1, "I217-V"},
{0x1534, 1, "I217-LM"},
{0x153A, 1, "I218-V"},
{0x153B, 1, "I218-LM"},
{0x156F, 1, "I219-V"},
{0x1570, 1, "I219-LM"},
{0x157B, 1, "I219-V-2"},
{0x157A, 1, "I219-LM-2"},
{0x15D3, 1, "I219-V-3"},
{0x15D8, 1, "I219-LM-3"},
{0x15E3, 1, "I219-V-4"},
{0x15D6, 1, "I219-V-5"},
{0x15BD, 1, "I219-V-6"},
{0x15B8, 1, "I219-LM-4"},
{0x15BB, 1, "I219-LM-5"},
{0x15DF, 1, "I219-LM-6"},
{0x15E0, 1, "I219-V-7"},
{0x15E1, 1, "I219-LM-7"},
{0x15E2, 1, "I219-LM-8"},
{0x15E4, 1, "I219-V-8"},
{0x15E5, 1, "I219-V-9"},
{0x15F9, 1, "I219-LM-9"},
{0x15FA, 1, "I219-V-10"},
{0x15FB, 1, "I219-LM-10"},
{0x15FC, 1, "I219-LM-11"},
{0x15FD, 1, "I219-V-11"},
{0x15FE, 1, "I219-LM-12"},
{0x15FF, 1, "I219-V-12"},
/* I210/I211 (igb 系列) — 与 e1000e MMIO 兼容 */
{0x1537, 1, "I210"},
{0x1536, 1, "I211-AT"},
/* 82575/82576/82580 (igb 系列, 多队列, 基本兼容) */
{0x10A7, 1, "82575EB"},
{0x10A9, 1, "82575EB-SerDes"},
{0x10C9, 1, "82576"},
{0x10E6, 1, "82576-Fiber"},
{0x10E7, 1, "82576-Quad"},
{0x150D, 1, "82580"},
{0x1516, 1, "82580-Fiber"},
{0x1526, 1, "82576-2"},
{0, 0, 0} /* 终止符 */
};
struct e1000_rx_desc {
u64 addr;
@@ -89,6 +205,36 @@ struct e1000_tx_desc {
u16 special;
} __attribute__((packed));
/* ---- 统计计数器（用于真机诊断） ---- */
static u64 g_stat_tx_ok; /* TX 成功帧数 */
static u64 g_stat_tx_err; /* TX 失败帧数 */
static u64 g_stat_tx_timeout; /* TX 超时次数 */
static u64 g_stat_rx_ok; /* RX 成功帧数 */
static u64 g_stat_rx_err; /* RX 错误帧数 */
static u64 g_stat_rx_overflow; /* RX buffer 太小丢包 */
static u64 g_stat_irq_count; /* IRQ 中断次数 */
/* 前向声明 */
static void raw_log(const char *s);
static void raw_hex(const char *prefix, u64 v);
/* 每 256 次数据路径事件打印一行摘要（避免刷串口）。 */
static void net_stat_tick(void) {
u64 sum = g_stat_tx_ok + g_stat_tx_err + g_stat_tx_timeout +
g_stat_rx_ok + g_stat_rx_err + g_stat_rx_overflow;
if ((sum & 0xFF) != 0) return;
raw_log("[e1000.raw] stat tx_ok=");
raw_hex("", g_stat_tx_ok);
raw_log("[e1000.raw] stat tx_err=");
raw_hex("", g_stat_tx_err);
raw_log("[e1000.raw] stat rx_ok=");
raw_hex("", g_stat_rx_ok);
raw_log("[e1000.raw] stat rx_err=");
raw_hex("", g_stat_rx_err);
raw_log("[e1000.raw] stat rx_overflow=");
raw_hex("", g_stat_rx_overflow);
}
static const struct dkm_log_api *g_log;
static volatile u32 *g_mmio;
static u64 g_bar0_phys;
@@ -103,6 +249,8 @@ static u64 g_tx_buf_phys[E1000_TX_DESC_COUNT];
static u32 g_rx_tail;
static u32 g_tx_tail;
static int g_rings_ready;
static u8 g_device_family; /* 0=classic e1000, 1=e1000e */
static u16 g_device_id; /* PCI device ID */
#define COM1 0x3F8
@@ -132,6 +280,23 @@ static void raw_hex(const char *prefix, u64 v) {
raw_log("\n");
}
/* 打印 MAC 地址 */
static void raw_mac(const char *prefix, const u8 mac[6]) {
static const char h[] = "0123456789abcdef";
raw_log(prefix);
for (int i = 0; i < 6; i++) {
serial_wait_tx();
dkm_outb(COM1, (u8)h[(mac[i] >> 4) & 0xf]);
serial_wait_tx();
dkm_outb(COM1, (u8)h[mac[i] & 0xf]);
if (i < 5) {
serial_wait_tx();
dkm_outb(COM1, ':');
}
}
raw_log("\n");
}
static u32 pci_read(u8 bus, u8 dev, u8 func, u8 reg) {
return dkm_pci_read(bus, dev, func, reg);
}
@@ -140,40 +305,165 @@ static void pci_write(u8 bus, u8 dev, u8 func, u8 reg, u32 v) {
dkm_pci_write(bus, dev, func, reg, v);
}
static int e1000_find(u8 *bus, u8 *dev) {
for (u8 d=0; d<32; d++) {
u32 vd = pci_read(0,d,0,PCI_VENDOR_ID);
if ((vd&0xffff)==0xffff) continue;
if ((vd&0xffff)!=0x8086) continue;
u16 did = (u16)(vd>>16);
/* 82540EM(0x100E) / 82545 等经典 e1000，以及 82574L(0x10D3) e1000e */
if (did==0x100E || did==0x10D3 || did==0x1004 || did==0x100F ||
did==0x10D3 || did==0x153A || did==0x1533) {
*bus=0; *dev=d; return 0;
/* 根据 PCI Device ID 查找设备信息 */
static const struct e1000_device_id *e1000_lookup_id(u16 did) {
for (int i = 0; g_e1000_ids[i].did != 0; i++) {
if (g_e1000_ids[i].did == did) return &g_e1000_ids[i];
}
return 0;
}
/* 多总线扫描 e1000 设备。真机网卡可能在 bus>0 上。 */
static int e1000_find(u8 *out_bus, u8 *out_dev, u8 *out_func,
const struct e1000_device_id **out_id) {
for (u8 bus = 0; bus < 16; bus++) {
for (u8 dev = 0; dev < 32; dev++) {
u32 vd = pci_read(bus, dev, 0, PCI_VENDOR_ID);
if ((vd & 0xffff) == 0xffff) continue;
if ((vd & 0xffff) != 0x8086) continue;
/* 检查是否为多功能设备 */
u8 header = (u8)(pci_read(bus, dev, 0, PCI_HEADER) >> 16);
u8 func_count = (header & 0x80) ? 8 : 1;
for (u8 func = 0; func < func_count; func++) {
u32 vd2 = pci_read(bus, dev, func, PCI_VENDOR_ID);
if ((vd2 & 0xffff) == 0xffff) continue;
if ((vd2 & 0xffff) != 0x8086) continue;
u16 did = (u16)(vd2 >> 16);
const struct e1000_device_id *id = e1000_lookup_id(did);
if (id) {
*out_bus = bus;
*out_dev = dev;
*out_func = func;
if (out_id) *out_id = id;
return 0;
}
}
}
}
return -1;
}
/* 读取 EEPROM 一个 16-bit 字（经典 e1000 EERD 接口）。失败返回 0xffff。 */
/* 读取 EEPROM 一个 16-bit 字（经典 e1000 EERD 接口）。
* 经典 e1000: EERD bit0=START, addr<<8, 完成后 bit4=DONE, data 在高 16 位。
* e1000e: EERD 接口类似但地址字段位置不同 (addr<<2, bit1=START, bit2=DONE)。
* 失败返回 0xffff。 */
static u16 e1000_eeprom_read(u16 addr) {
/* EERD: bit0=START, addr<<8, 完成后 bit4=DONE, data 在高 16 位 */
g_mmio[E1000_EERD/4] = ((u32)addr << 8) | 1u;
/* 实机: 100ms TSC 超时 */
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 100;
while (dkm_rdtsc() < deadline) {
u32 v = g_mmio[E1000_EERD/4];
if (v & (1u<<4)) return (u16)(v >> 16);
__asm__ volatile("pause");
if (g_device_family == 1) {
/* e1000e: EERD addr<<2 | START(bit1), DONE=bit2, data=bits[31:16] */
g_mmio[E1000_EERD/4] = ((u32)addr << 2) | 0x2u;
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 100;
while (dkm_rdtsc() < deadline) {
u32 v = g_mmio[E1000_EERD/4];
if (v & (1u << 2)) return (u16)(v >> 16);
__asm__ volatile("pause");
}
} else {
/* 经典 e1000: EERD addr<<8 | START(bit0), DONE=bit4 */
g_mmio[E1000_EERD/4] = ((u32)addr << 8) | 1u;
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 100;
while (dkm_rdtsc() < deadline) {
u32 v = g_mmio[E1000_EERD/4];
if (v & (1u << 4)) return (u16)(v >> 16);
__asm__ volatile("pause");
}
}
return 0xffff;
}
/* 检测链路状态，返回 0=down, 1=up */
static int e1000_link_up(void) {
if (!g_mmio) return 0;
return (g_mmio[E1000_STATUS/4] & E1000_STATUS_LU) ? 1 : 0;
}
/* 读取当前速度 (Mbps)，返回 10/100/1000/0(未知) */
static u32 e1000_link_speed(void) {
if (!g_mmio) return 0;
u32 status = g_mmio[E1000_STATUS/4];
u32 speed_bits = status & E1000_STATUS_SPEEDMASK;
if (g_device_family == 1) {
/* e1000e speed encoding: 00=reserved, 01=100, 10=1000, 11=1000 */
if (speed_bits == E1000_STATUS_SPEED_1000_E1000E ||
speed_bits == (2u << 6))
return 1000;
if (speed_bits == E1000_STATUS_SPEED_100) return 100;
if (speed_bits == E1000_STATUS_SPEED_10) return 10;
return 0;
} else {
/* 经典 e1000 speed encoding: 00=10, 01=100, 10=1000 */
if (speed_bits == E1000_STATUS_SPEED_1000) return 1000;
if (speed_bits == E1000_STATUS_SPEED_100) return 100;
if (speed_bits == E1000_STATUS_SPEED_10) return 10;
return 0;
}
}
/* 读取当前双工模式，返回 1=全双工, 0=半双工 */
static int e1000_link_full_duplex(void) {
if (!g_mmio) return 0;
return (g_mmio[E1000_STATUS/4] & E1000_STATUS_FD) ? 1 : 0;
}
/* 打印链路诊断信息 */
static void e1000_log_link_status(void) {
int up = e1000_link_up();
if (up) {
u32 speed = e1000_link_speed();
int fd = e1000_link_full_duplex();
const char *speed_str = "unknown";
char speed_buf[8];
if (speed == 1000) speed_str = "1000";
else if (speed == 100) speed_str = "100";
else if (speed == 10) speed_str = "10";
else {
/* 手动转数字 */
speed_buf[0] = '0' + (char)(speed / 1000 % 10);
speed_buf[1] = '0' + (char)(speed / 100 % 10);
speed_buf[2] = '0' + (char)(speed / 10 % 10);
speed_buf[3] = '0' + (char)(speed % 10);
speed_buf[4] = 0;
speed_str = speed_buf;
}
raw_log("[e1000] link=up speed=");
raw_log(speed_str);
raw_log("Mbps duplex=");
raw_log(fd ? "full" : "half");
raw_log("\n");
} else {
raw_log("[e1000] link=down\n");
}
raw_hex("[e1000] STATUS=", g_mmio ? g_mmio[E1000_STATUS/4] : 0);
}
/* 等待链路就绪（最长 2 秒），实机 PHY 自动协商可能需要时间 */
static int e1000_wait_for_link(void) {
if (!g_mmio) return -1;
/* 设置 ASDE (Auto-Speed Detect Enable) 让 PHY 自动协商 */
if (g_device_family == 1) {
/* e1000e: 建议先清除 FRCSPD/FRCDPLX，让自动协商生效 */
u32 ctrl = g_mmio[E1000_CTRL/4];
ctrl &= ~(E1000_CTRL_FRCSPD | E1000_CTRL_FRCDPLX);
ctrl |= E1000_CTRL_SLU | E1000_CTRL_ASDE;
g_mmio[E1000_CTRL/4] = ctrl;
}
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 2000;
while (dkm_rdtsc() < deadline) {
if (e1000_link_up()) return 0;
__asm__ volatile("pause");
}
return -1; /* 链路未就绪但驱动继续初始化（后续可能 link up） */
}
static int irq_handler(u8 irq) {
(void)irq;
if (!g_mmio) return 0;
u32 icr = g_mmio[E1000_ICR/4];
if (icr) {
g_stat_irq_count++;
DKM_STAT_INC(e1k_irq_count);
g_mmio[E1000_ICR/4] = icr; /* clear */
}
return 0;
@@ -228,7 +518,7 @@ static int e1000_init_rings(const struct dkm_dma_api *dma) {
g_tx_desc[i].status = E1000_TX_STA_DD;
}
dma_fence();
raw_log("[e1000.raw] ring/buffers allocated\n");
raw_log("[e1000] ring/buffers allocated\n");
g_mmio[E1000_RCTL/4] = 0;
g_mmio[E1000_TCTL/4] = 0;
@@ -259,19 +549,23 @@ static int e1000_init_rings(const struct dkm_dma_api *dma) {
mmio_flush();
g_rings_ready = 1;
raw_log("[e1000.raw] RX/TX rings initialized\n");
raw_log("[e1000] RX/TX rings initialized\n");
return 0;
}
static int e1000_tx(void *ctx, const void *packet, u32 length) {
(void)ctx;
raw_log("[e1000.raw] tx enter\n");
if (!g_rings_ready || !packet || length == 0 || length > E1000_TX_BUF_SIZE) return -1;
if (!g_rings_ready || !packet || length == 0 || length > E1000_TX_BUF_SIZE) {
g_stat_tx_err++;
return -1;
}
u32 index = g_tx_tail;
if (!(g_tx_desc[index].status & E1000_TX_STA_DD)) return -2;
if (!(g_tx_desc[index].status & E1000_TX_STA_DD)) {
g_stat_tx_err++;
return -2; /* TX ring full */
}
copy_bytes(g_tx_buf[index], packet, length);
dma_fence();
raw_log("[e1000.raw] tx copied\n");
g_tx_desc[index].length = (u16)length;
g_tx_desc[index].cso = 0;
g_tx_desc[index].cmd = E1000_TX_CMD_EOP | E1000_TX_CMD_IFCS | E1000_TX_CMD_RS;
@@ -287,17 +581,20 @@ static int e1000_tx(void *ctx, const void *packet, u32 length) {
while (dkm_rdtsc() < deadline) {
dma_fence();
if (g_tx_desc[index].status & E1000_TX_STA_DD) {
raw_log("[e1000.raw] tx complete\n");
g_stat_tx_ok++;
DKM_STAT_INC(e1k_tx_packets);
net_stat_tick();
return 0;
}
__asm__ volatile("pause");
}
raw_log("[e1000.raw] tx timeout\n");
raw_hex("[e1000.raw] STATUS=", g_mmio[E1000_STATUS/4]);
raw_hex("[e1000.raw] TCTL=", g_mmio[E1000_TCTL/4]);
raw_hex("[e1000.raw] TDH=", g_mmio[E1000_TDH/4]);
raw_hex("[e1000.raw] TDT=", g_mmio[E1000_TDT/4]);
raw_hex("[e1000.raw] desc.status=", g_tx_desc[index].status);
g_stat_tx_timeout++;
g_stat_tx_err++;
raw_log("[e1000] tx timeout\n");
raw_hex("[e1000] STATUS=", g_mmio[E1000_STATUS/4]);
raw_hex("[e1000] TCTL=", g_mmio[E1000_TCTL/4]);
raw_hex("[e1000] TDH=", g_mmio[E1000_TDH/4]);
raw_hex("[e1000] TDT=", g_mmio[E1000_TDT/4]);
return -3;
}
@@ -307,9 +604,10 @@ static int e1000_rx_poll(void *ctx, void *buffer, u32 capacity, u32 *out_length)
if (!g_rings_ready || !buffer || !out_length) return -1;
u32 next = (g_rx_tail + 1) % E1000_RX_DESC_COUNT;
dma_fence();
if (!(g_rx_desc[next].status & E1000_RX_STA_DD)) return 1;
if (!(g_rx_desc[next].status & E1000_RX_STA_DD)) return 1; /* no data */
u32 len = g_rx_desc[next].length;
if (len > capacity) {
g_stat_rx_overflow++;
g_rx_desc[next].status = 0;
dma_fence();
g_rx_tail = next;
@@ -324,6 +622,9 @@ static int e1000_rx_poll(void *ctx, void *buffer, u32 capacity, u32 *out_length)
g_rx_tail = next;
g_mmio[E1000_RDT/4] = g_rx_tail;
mmio_flush();
g_stat_rx_ok++;
DKM_STAT_INC(e1k_rx_packets);
net_stat_tick();
return 0;
}
@@ -332,58 +633,81 @@ int driver_init(const struct dkm_kernel_api *api,
struct dkm_driver_handle *handle) {
/* 实机要求: 先校准 TSC, 再使用基于 CPU 频率的延迟 */
dkm_tsc_calibrate();
raw_log("[e1000.raw] driver_init entered\n");
raw_log("[e1000] driver_init entered\n");
(void)handle;
if (!api||!api->log) return -1;
g_log=api->log;
dkm_instr_init(api);
raw_log("[e1000.raw] before find\n");
u8 bus, dev;
if (e1000_find(&bus,&dev)!=0) {
raw_log("[e1000.raw] device not found\n");
u8 bus, dev, func;
const struct e1000_device_id *dev_id = 0;
if (e1000_find(&bus, &dev, &func, &dev_id) != 0) {
raw_log("[e1000] device not found\n");
return 0;
}
raw_log("[e1000.raw] found device\n");
g_device_id = (u16)(pci_read(bus, dev, func, PCI_VENDOR_ID) >> 16);
g_device_family = dev_id ? dev_id->family : 0;
raw_log("[e1000] found device: ");
raw_log(dev_id ? dev_id->name : "unknown");
raw_log(g_device_family == 1 ? " (e1000e)" : " (classic)");
raw_log("\n");
raw_hex("[e1000] PCI bus:dev:func=", ((u64)bus << 16) | ((u64)dev << 8) | func);
u32 cmd = pci_read(bus, dev, 0, PCI_COMMAND);
cmd |= 0x0007u; /* IO Space | Memory Space | Bus Master */
pci_write(bus, dev, 0, PCI_COMMAND, cmd);
raw_log("[e1000.raw] PCI command enabled\n");
/* 检查 PCI class code 确认是网络设备 (class=0x02) */
u32 class_reg = pci_read(bus, dev, func, 0x08);
u8 class_code = (u8)(class_reg >> 24);
if (class_code != 0x02) {
raw_log("[e1000] WARNING: PCI class is not Network\n");
}
u32 bar0 = pci_read(bus,dev,0,PCI_BAR0);
u32 cmd = pci_read(bus, dev, func, PCI_COMMAND);
cmd |= PCI_CMD_IO | PCI_CMD_MEM | PCI_CMD_BUSM;
pci_write(bus, dev, func, PCI_COMMAND, cmd);
raw_log("[e1000] PCI command enabled\n");
u32 bar0 = pci_read(bus, dev, func, PCI_BAR0);
if (bar0 & 1u) {
raw_log("[e1000.raw] IO BAR unsupported\n");
raw_log("[e1000] IO BAR unsupported\n");
return 0;
}
g_bar0_phys = (u64)(bar0 & 0xFFFFFFF0ULL);
if ((bar0 & 0x6u) == 0x4u) {
u32 bar1 = pci_read(bus,dev,0,PCI_BAR1);
u32 bar1 = pci_read(bus, dev, func, PCI_BAR1);
g_bar0_phys |= ((u64)bar1 << 32);
}
raw_hex("[e1000.raw] BAR0 raw=", bar0);
raw_hex("[e1000.raw] BAR0 phys=", g_bar0_phys);
raw_hex("[e1000] BAR0 phys=", g_bar0_phys);
if (!g_bar0_phys) {
raw_log("[e1000.raw] BAR0 unavailable\n");
raw_log("[e1000] BAR0 unavailable\n");
return 0;
}
raw_log("[e1000.raw] BAR0 ok\n");
/* 真机 BAR0 可能在 4G 以上（如 I219），需 mm_map_mmio 检查 */
u64 hhdm = api->hhdm_offset;
g_mmio = (volatile u32*)(uintptr_t)(hhdm + g_bar0_phys);
(void)g_mmio[E1000_STATUS/4];
raw_log("[e1000.raw] MMIO status read ok\n");
/* 尝试读取 STATUS 寄存器验证映射是否有效 */
u32 status_val = g_mmio[E1000_STATUS/4];
(void)status_val;
raw_log("[e1000] MMIO status read ok\n");
/* 软件复位，确保 RX/TX/RAL/RAH 从已知状态开始配置。 */
g_mmio[E1000_CTRL/4] = g_mmio[E1000_CTRL/4] | (1u << 26);
g_mmio[E1000_CTRL/4] = g_mmio[E1000_CTRL/4] | E1000_CTRL_RST;
/* 实机: 1秒 TSC 超时等待复位完成 */
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 1000;
while (dkm_rdtsc() < deadline) {
if (!(g_mmio[E1000_CTRL/4] & (1u << 26))) break;
if (!(g_mmio[E1000_CTRL/4] & E1000_CTRL_RST)) break;
__asm__ volatile("pause");
}
raw_log("[e1000.raw] reset done\n");
if (g_mmio[E1000_CTRL/4] & E1000_CTRL_RST) {
raw_log("[e1000] WARNING: reset did not complete\n");
} else {
raw_log("[e1000] reset done\n");
}
/* 等待 EEPROM 完成（实机复位后 EEPROM 状态机需要时间） */
dkm_delay_ms(1);
/* 读 MAC 地址：先尝试 RAL/RAH，空时走 EEPROM */
u32 ral = g_mmio[E1000_RAL/4];
u32 rah = g_mmio[E1000_RAH/4];
u8 mac[6];
@@ -403,60 +727,102 @@ int driver_init(const struct dkm_kernel_api *api,
mac[0]=(u8)(w0&0xff); mac[1]=(u8)(w0>>8);
mac[2]=(u8)(w1&0xff); mac[3]=(u8)(w1>>8);
mac[4]=(u8)(w2&0xff); mac[5]=(u8)(w2>>8);
raw_log("[e1000] MAC from EEPROM\n");
} else {
raw_log("[e1000] EEPROM read failed (all 0xffff)\n");
}
} else {
raw_log("[e1000] MAC from RAL/RAH\n");
}
/* 仍为全 0 时使用与虚拟机配置一致的回退 MAC，保证收发地址匹配 */
if (mac[0]==0&&mac[1]==0&&mac[2]==0&&mac[3]==0&&mac[4]==0&&mac[5]==0) {
/* 仍为全 0 或多播位错误时使用与虚拟机配置一致的回退 MAC */
if ((mac[0]==0&&mac[1]==0&&mac[2]==0&&mac[3]==0&&mac[4]==0&&mac[5]==0) ||
(mac[0] & 1)) {
mac[0]=0x52; mac[1]=0x54; mac[2]=0x00; mac[3]=0x12; mac[4]=0x34; mac[5]=0x56;
raw_log("[e1000.raw] using fallback MAC 52:54:00:12:34:56\n");
raw_log("[e1000] using fallback MAC 52:54:00:12:34:56\n");
}
raw_mac("[e1000] MAC=", mac);
/* 写入 RAL/RAH 并置 AV(bit31) 使能接收该单播地址 */
g_mmio[E1000_RAL/4] = (u32)mac[0] | ((u32)mac[1]<<8) | ((u32)mac[2]<<16) | ((u32)mac[3]<<24);
g_mmio[E1000_RAH/4] = (u32)mac[4] | ((u32)mac[5]<<8) | (1u<<31);
raw_log("[e1000.raw] MAC read ok\n");
dma_fence();
u32 irq_line = pci_read(bus,dev,0,PCI_IRQ_LINE) & 0xff;
u32 irq_line = pci_read(bus, dev, func, PCI_IRQ_LINE) & 0xff;
raw_hex("[e1000] IRQ line=", irq_line);
/* 设置链路：FD + SLU，并允许自动协商 */
g_mmio[E1000_CTRL/4] = g_mmio[E1000_CTRL/4] | E1000_CTRL_FD | E1000_CTRL_SLU;
raw_log("[e1000.raw] link setup\n");
mmio_flush();
g_mmio[E1000_IMC/4] = 0xffffffffu;
(void)g_mmio[E1000_ICR/4];
raw_log("[e1000.raw] interrupts masked\n");
raw_log("[e1000] interrupts masked\n");
/* 等待链路就绪（实机 PHY 自动协商可能需要 1-2 秒） */
int link_rc = e1000_wait_for_link();
if (link_rc == 0) {
raw_log("[e1000] link up after auto-negotiation\n");
} else {
raw_log("[e1000] link not yet up (will retry during operation)\n");
}
/* 打印链路诊断 */
e1000_log_link_status();
int ring_rc = e1000_init_rings(api->dma);
if (ring_rc != 0) raw_log("[e1000.raw] ring init failed\n");
if (ring_rc != 0) {
raw_log("[e1000] ring init failed\n");
raw_hex("[e1000] ring_rc=", (u64)(i32)ring_rc);
}
/* 根据链路状态设置 netdev flags */
u32 netdev_flags = 0;
if (e1000_link_up()) netdev_flags |= DKM_NET_F_LINK_UP;
if (g_rings_ready) netdev_flags |= DKM_NET_F_TX_READY | DKM_NET_F_RX_READY;
if (api->net && api->net->register_device) {
struct dkm_net_device_desc netdev;
zero_bytes(&netdev, sizeof(netdev));
netdev.name = "e1000";
for (int i=0;i<6;i++) netdev.mac[i] = mac[i];
netdev.flags = DKM_NET_F_LINK_UP;
if (g_rings_ready) netdev.flags |= DKM_NET_F_TX_READY | DKM_NET_F_RX_READY;
netdev.flags = netdev_flags;
netdev.ctx = 0;
netdev.tx = g_rings_ready ? e1000_tx : 0;
netdev.rx_poll = g_rings_ready ? e1000_rx_poll : 0;
int net_index = api->net->register_device(&netdev);
(void)net_index;
raw_log("[e1000.raw] netdev registered\n");
raw_hex("[e1000] netdev registered, index=", (u64)(i32)net_index);
} else {
raw_log("[e1000.raw] net API unavailable\n");
raw_log("[e1000] net API unavailable\n");
}
g_mmio[E1000_IMC/4] = 0xffffffffu;
if (api->irq_register && irq_line < 16) {
if (api->irq_register && irq_line < 16 && irq_line > 0) {
api->irq_register((u8)irq_line, (void*)(uintptr_t)irq_handler);
raw_log("[e1000.raw] IRQ registered\n");
raw_log("[e1000] IRQ registered\n");
} else {
raw_log("[e1000] IRQ not registered (polling mode)\n");
}
raw_log("[e1000.raw] driver ready\n");
/* 最终诊断快照 */
e1000_log_link_status();
raw_log("[e1000] driver ready\n");
return 0;
}
__attribute__((visibility("default")))
int driver_exit(struct dkm_driver_handle *handle) {
(void)handle; return 0;
(void)handle;
/* 关闭 RX/TX，释放 rings */
if (g_mmio) {
g_mmio[E1000_RCTL/4] = 0;
g_mmio[E1000_TCTL/4] = 0;
g_mmio[E1000_IMC/4] = 0xffffffffu;
mmio_flush();
}
g_rings_ready = 0;
g_mmio = 0;
return 0;
}
CODE/DKM/fat32/fat32.c
+10 -52
View File
@@ -6,60 +6,15 @@
#include <stdint.h>
#include "../dkm_shared.h"
#include "../dkm_instr.h"
DKM_STAT_DECL(fat0_reads);
DKM_STAT_DECL(fat0_fat_walks);
/* ---------------------------------------------------------------
* DKM ABI types
* FAT32-specific types (DKM ABI types come from dkm_shared.h)
* --------------------------------------------------------------- */
#define DKM_DRIVER_MAGIC 0x444B4D31u
#define DKM_ABI_VERSION 1u
typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned int u32;
typedef unsigned long long u64;
typedef long long i64;
#define NULL ((void *)0)
struct dkm_log_api {
void (*info)(const char *msg);
void (*warn)(const char *msg);
void (*error)(const char *msg);
void (*panic)(const char *msg);
};
struct dkm_block_api {
int (*register_device)(const void *desc);
u32 (*device_count)(void);
int (*read)(u32 index, u64 lba, u32 count, void *buffer);
int (*write)(u32 index, u64 lba, u32 count, const void *buffer);
u64 (*sector_size)(u32 index);
const char *(*device_name)(u32 index);
int (*set_write_fn)(u32 index, int (*fn)(void *ctx, u64 lba, u32 count, const void *buf));
};
struct dkm_kernel_api {
u32 version; u32 size; u64 feature_bits;
const struct dkm_log_api *log;
const void *mem,*utsm,*irq,*pci_api,*dma,*vfs_api,*net,*timer,*drr;
const void *rsdp_address;
const void *fb_address;
u64 fb_width,fb_height,fb_pitch; u16 fb_bpp;
const void *boot_modules_response;
int (*irq_register)(u8 irq, void *handler);
u64 hhdm_offset;
const struct dkm_block_api *block;
};
struct dkm_driver_handle;
struct dkm_driver_desc {
u32 magic; u16 abi_version; u16 desc_size;
const char *name,*version,*vendor;
u32 driver_class,stage,flags,priority;
const char *const *depends; u32 depends_count;
const char *const *provides; u32 provides_count;
u64 min_kernel_abi,feature_bits,reserved0,reserved1;
};
struct limine_file {
u64 revision; void *address; u64 size; char *path; char *cmdline; u8 _pad[56];
@@ -237,6 +192,7 @@ int driver_init(const struct dkm_kernel_api *api,
(void)handle;
if (!api||!api->log) return -1;
g_log=api->log;
dkm_instr_init(api);
u64 disk_size = 0;
const u8 *disk = fat32_select_disk_image(api, &disk_size);
@@ -279,6 +235,7 @@ int driver_init(const struct dkm_kernel_api *api,
/* try to read README.TXT content */
if (strneq(dir[e].name,"README TXT",11)) {
DKM_STAT_INC(fat0_reads);
u32 f_clus = read_u16_le((const u8*)&dir[e].cluster_low);
u32 f_off = data_start + (f_clus - 2)*cluster_size;
u32 f_size = dir[e].file_size;
@@ -297,6 +254,7 @@ int driver_init(const struct dkm_kernel_api *api,
/* read next cluster from FAT */
u32 fat_ent_off = fat_start + clus * 4;
clus = read_u32_le(disk + fat_ent_off) & 0x0FFFFFFF;
DKM_STAT_INC(fat0_fat_walks);
}
g_log->info("[fat32] driver ready");
CODE/DKM/nvme/nvme.c
+802 -208
View File
@@ -1,76 +1,27 @@
/* DKM NVMe Driver — PCI NVMe discovery and controller register enumeration
/* DKM NVMe Driver — 高位 BAR MMIO 映射 + admin/IO queue + block provider
*
* Stage 1 optional storage driver, depends on "pci" and "irq", provides "block".
* This first version only discovers NVMe controllers and reads BAR0 registers;
* it does not enable queues or submit admin commands.
*
* 实现要点:
* 1. 64-bit BAR 常位于 4G 以上（QEMU/OVMF 把 NVMe BAR0 放到 ~48GiB），
* HHDM 直映射不保证覆盖。访问前先用 kernel_api.mmio->is_mapped 做
* 四级页表 walk 核实；未映射则 map_mmio 原地补建 4KiB UC 页。
* 2. 同步轮询模型: 单 outstanding 命令 + CQE phase bit 轮询 + TSC 超时,
* 不依赖 IRQ/MSI（当前 PIC/APIC 后端未完工）。
* 3. 数据面: 32 页 (128KiB) 物理连续 DMA bounce buffer + 单页 PRP list。
* PRP 布局按传输页数分档: 1 页单 PRP1; 2 页 PRP2 直连第二页;
* >=3 页 PRP2 指向 PRP list 页 (512 项 u64, 上限 513 页 = 2052KiB)。
* chunk 上限 = min(buffer 页数, 2^MDTS 页, NLB 16-bit)。
* 读写一律 bounce: 调用方缓冲是任意虚拟地址 (DSK BSS / vmm dbuf),
* 物理底账未知, 统一经 DMA 缓冲中转, 一次命令读满 chunk 再 memcpy。
* WRITE 路径 memcpy 后 wbinvd (沿用 ahci BUG-P0-2 防陈旧 cache 先例),
* 每次写调用末尾随一个 FLUSH。admin/IO queue 各 64 项, 均每队列一页。
*/
#include <stdint.h>
#include "../dkm_shared.h"
#include "../dkm_instr.h"
#define DKM_DRIVER_MAGIC 0x444B4D31u
#define DKM_ABI_VERSION 1u
typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned int u32;
typedef unsigned long long u64;
typedef long long i64;
#define NULL ((void *)0)
struct dkm_log_api {
void (*info)(const char *msg);
void (*warn)(const char *msg);
void (*error)(const char *msg);
void (*panic)(const char *msg);
};
struct dkm_kernel_api {
u32 version;
u32 size;
u64 feature_bits;
const struct dkm_log_api *log;
const void *mem;
const void *utsm;
const void *irq;
const void *pci;
const void *dma;
const void *vfs;
const void *net;
const void *timer;
const void *drr;
const void *rsdp_address;
const void *fb_address;
u64 fb_width;
u64 fb_height;
u64 fb_pitch;
u16 fb_bpp;
const void *boot_modules_response;
int (*irq_register)(u8 irq, void *handler);
u64 hhdm_offset;
};
struct dkm_driver_handle;
struct dkm_driver_desc {
u32 magic;
u16 abi_version;
u16 desc_size;
const char *name;
const char *version;
const char *vendor;
u32 driver_class;
u32 stage;
u32 flags;
u32 priority;
const char *const *depends;
u32 depends_count;
const char *const *provides;
u32 provides_count;
u64 min_kernel_abi;
u64 feature_bits;
u64 reserved0;
u64 reserved1;
};
DKM_STAT_DECL(nvm0_pci_found);
static const char *const g_depends[] = { "pci", "irq" };
static const char *const g_provides[] = { "block" };
@@ -81,7 +32,7 @@ const struct dkm_driver_desc driver_desc = {
.abi_version = DKM_ABI_VERSION,
.desc_size = sizeof(struct dkm_driver_desc),
.name = "nvme",
.version = "0.1.0",
.version = "0.3.0",
.vendor = "Deshab",
.driver_class = 6, /* DKM_CLASS_STORAGE */
.stage = 1,
@@ -97,78 +48,69 @@ const struct dkm_driver_desc driver_desc = {
.reserved1 = 0
};
#define PCI_ADDR 0xCF8
#define PCI_DATA 0xCFC
#define PCI_VENDOR_ID 0x00
#define PCI_COMMAND 0x04
#define PCI_CLASS_REG 0x08
#define PCI_HEADER 0x0E
#define PCI_BAR0 0x10
#define PCI_BAR1 0x14
#define PCI_IRQ_LINE 0x3C
#define PCI_CMD_MEM (1u << 1)
#define PCI_CMD_BUSM (1u << 2)
/* ---- NVMe 控制器寄存器 ---- */
#define NVME_CAP 0x00 /* u64 capabilities */
#define NVME_VS 0x08 /* u32 version */
#define NVME_INTMS 0x0C /* interrupt mask set */
#define NVME_INTMC 0x10 /* interrupt mask clear */
#define NVME_CC 0x14 /* controller configuration */
#define NVME_CSTS 0x1C /* controller status */
#define NVME_AQA 0x24 /* admin queue attributes */
#define NVME_ASQ 0x28 /* u64 admin SQ base */
#define NVME_ACQ 0x30 /* u64 admin CQ base */
#define NVME_DB_BASE 0x1000 /* doorbell 起始偏移 */
#define NVME_CLASS_STORAGE 0x01
#define NVME_SUBCLASS_NVM 0x08
#define NVME_PROGIF_NVME 0x02
#define NVME_CAP 0x00
#define NVME_VS 0x08
#define NVME_INTMS 0x0C
#define NVME_INTMC 0x10
#define NVME_CC 0x14
#define NVME_CSTS 0x1C
#define NVME_NSSR 0x20
#define NVME_AQA 0x24
#define NVME_ASQ 0x28
#define NVME_ACQ 0x30
#define NVME_CMBLOC 0x38
#define NVME_CMBSZ 0x3C
/* admin 命令 opcode */
#define NVME_ADM_DELETE_IOSQ 0x00
#define NVME_ADM_CREATE_IOSQ 0x01
#define NVME_ADM_DELETE_IOCQ 0x04
#define NVME_ADM_CREATE_IOCQ 0x05
#define NVME_ADM_IDENTIFY 0x06
/* IO 命令 opcode（NVM 命令集: 00h=Flush, 01h=Write, 02h=Read） */
#define NVME_IO_FLUSH 0x00
#define NVME_IO_READ 0x02
#define NVME_IO_WRITE 0x01
#define ADMIN_QSIZE 64 /* entry 数（SQ 64*64B=4K 一页, CQ 64*16B=1K） */
#define IO_QSIZE 64
#define NVME_IDENT_BYTES 4096 /* identify 数据页 */
#define NVME_DATA_PAGES 32u /* IO bounce: 128KiB = 512B×256 扇区,
* 正好覆盖 DSK FAT32 的 256 扇区批量读 */
#define NVME_PRP_LIST_ENTRIES 512u /* 单 list 页 4KiB / 8B = 512 项 */
static const struct dkm_log_api *g_log;
static volatile u8 *g_regs;
static volatile u8 *g_regs; /* MMIO 基址（虚拟） */
static u32 g_db_stride; /* doorbell 步长 = 4 << CAP.DSTRD */
static u64 g_timeout_ms; /* CAP.TO * 500ms */
static __inline__ void outl(u16 port, u32 value) {
__asm__ volatile ("outl %0, %1" :: "a"(value), "Nd"(port));
}
static struct dkm_dma_buffer g_asq; /* admin SQ */
static struct dkm_dma_buffer g_acq; /* admin CQ */
static struct dkm_dma_buffer g_ident; /* identify 数据页 */
static struct dkm_dma_buffer g_data; /* IO 读写 bounce (NVME_DATA_PAGES 页连续) */
static struct dkm_dma_buffer g_prp; /* PRP list 页 */
static struct dkm_dma_buffer g_iosq; /* IO SQ (qid=1) */
static struct dkm_dma_buffer g_iocq; /* IO CQ (qid=1) */
static __inline__ u32 inl(u16 port) {
u32 value;
__asm__ volatile ("inl %1, %0" : "=a"(value) : "Nd"(port));
return value;
}
static u32 g_asq_tail;
static u32 g_acq_head;
static u32 g_acq_phase;
static u32 g_iosq_tail;
static u32 g_iocq_head;
static u32 g_iocq_phase;
static u16 g_cid;
static u32 pci_read(u8 bus, u8 dev, u8 func, u8 reg) {
u32 addr = (1u << 31)
| ((u32)bus << 16)
| ((u32)dev << 11)
| ((u32)func << 8)
| ((u32)reg & 0xFC);
outl(PCI_ADDR, addr);
return inl(PCI_DATA);
}
static u64 g_sector_size;
static u64 g_sector_count;
static u32 g_mdts; /* Identify Controller byte 77 */
static u32 g_chunk_max_sectors; /* 单命令扇区上限 (buffer/MDTS/NLB 取小) */
static int g_io_ready;
static int g_test_disk; /* LBA0 带 DESHABNVME0 签名的测试盘 */
static void pci_write(u8 bus, u8 dev, u8 func, u8 reg, u32 value) {
u32 addr = (1u << 31)
| ((u32)bus << 16)
| ((u32)dev << 11)
| ((u32)func << 8)
| ((u32)reg & 0xFC);
outl(PCI_ADDR, addr);
outl(PCI_DATA, value);
}
static u32 mmio_read32(u32 off) {
return *(volatile u32 *)(void *)(g_regs + off);
}
static u64 mmio_read64(u32 off) {
u32 lo = mmio_read32(off);
u32 hi = mmio_read32(off + 4);
return ((u64)hi << 32) | lo;
}
/* ---- 基础工具 ---- */
static void log_hex(const char *prefix, u64 value) {
static const char hex[] = "0123456789abcdef";
@@ -184,19 +126,66 @@ static void log_hex(const char *prefix, u64 value) {
g_log->info(buf);
}
/* 打印有界 ASCII 串（identify 数据中的 SN/MN 等定长字段） */
static void log_str(const char *prefix, const u8 *src, u32 len) {
char buf[48];
if (len > 40) len = 40;
for (u32 i = 0; i < len; i++) {
char c = (char)src[i];
buf[i] = (c >= 32 && c < 127) ? c : ' ';
}
buf[len] = 0;
g_log->info(prefix);
g_log->info(buf);
}
static void nvme_zero(void *ptr, u32 len) {
u8 *p = (u8 *)ptr;
for (u32 i = 0; i < len; i++) p[i] = 0;
}
static __inline__ void nvme_mb(void) {
/* x86 TSO 保证 store-store 顺序, 这里只需阻断编译器重排 */
__asm__ volatile("" ::: "memory");
}
static u32 mmio_read32(u32 off) {
return *(volatile u32 *)(void *)(g_regs + off);
}
static void mmio_write32(u32 off, u32 value) {
*(volatile u32 *)(void *)(g_regs + off) = value;
}
static u64 mmio_read64(u32 off) {
u32 lo = mmio_read32(off);
u32 hi = mmio_read32(off + 4);
return ((u64)hi << 32) | lo;
}
static void mmio_write64(u32 off, u64 value) {
mmio_write32(off, (u32)(value & 0xffffffffu));
mmio_write32(off + 4, (u32)(value >> 32));
}
static u32 sq_tail_db(u16 qid) { return NVME_DB_BASE + (2u * qid) * g_db_stride; }
static u32 cq_head_db(u16 qid) { return NVME_DB_BASE + (2u * qid + 1u) * g_db_stride; }
/* ---- PCI ---- */
static int nvme_find(u8 *out_bus, u8 *out_dev, u8 *out_func) {
for (u8 bus = 0; bus < 16; bus++) {
for (u8 dev = 0; dev < 32; dev++) {
u32 vd = pci_read(bus, dev, 0, PCI_VENDOR_ID);
u32 vd = dkm_pci_read(bus, dev, 0, PCI_VENDOR_ID);
if ((vd & 0xffff) == 0xffff) continue;
u8 header = (u8)(pci_read(bus, dev, 0, PCI_HEADER) >> 16);
u8 header = (u8)(dkm_pci_read(bus, dev, 0, PCI_HEADER) >> 16);
u8 func_count = (header & 0x80) ? 8 : 1;
for (u8 func = 0; func < func_count; func++) {
u32 vd2 = pci_read(bus, dev, func, PCI_VENDOR_ID);
u32 vd2 = dkm_pci_read(bus, dev, func, PCI_VENDOR_ID);
if ((vd2 & 0xffff) == 0xffff) continue;
u32 class_reg = pci_read(bus, dev, func, PCI_CLASS_REG);
u32 class_reg = dkm_pci_read(bus, dev, func, 0x08);
u8 prog_if = (u8)((class_reg >> 8) & 0xff);
u8 subclass = (u8)((class_reg >> 16) & 0xff);
u8 class_code = (u8)((class_reg >> 24) & 0xff);
@@ -215,37 +204,477 @@ static int nvme_find(u8 *out_bus, u8 *out_dev, u8 *out_func) {
return -1;
}
static void log_nvme_version(u32 vs) {
log_hex("[nvme] version major=", (vs >> 16) & 0xffff);
log_hex("[nvme] version minor=", (vs >> 8) & 0xff);
log_hex("[nvme] version tertiary=", vs & 0xff);
/* ---- CSTS 等待 ---- */
static int nvme_wait_csts(u32 mask, u32 expect, u64 timeout_ms) {
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * timeout_ms;
for (;;) {
u32 csts = mmio_read32(NVME_CSTS);
if (csts & (1u << 1)) { /* CFS: controller fatal status */
g_log->error("[nvme] CSTS.CFS set");
log_hex("[nvme] CSTS=", csts);
return -2;
}
if ((csts & mask) == expect) return 0;
if (dkm_rdtsc() > deadline) {
g_log->error("[nvme] CSTS wait timeout");
log_hex("[nvme] CSTS=", csts);
log_hex("[nvme] wait mask=", mask);
return -1;
}
__asm__ volatile("pause");
}
}
static void log_cap_fields(u64 cap) {
log_hex("[nvme] CAP.MQES=", cap & 0xffff);
log_hex("[nvme] CAP.CQR=", (cap >> 16) & 1);
log_hex("[nvme] CAP.AMS=", (cap >> 17) & 3);
log_hex("[nvme] CAP.TO=", (cap >> 24) & 0xff);
log_hex("[nvme] CAP.DSTRD=", (cap >> 32) & 0xf);
log_hex("[nvme] CAP.NSSRS=", (cap >> 36) & 1);
log_hex("[nvme] CAP.CSS=", (cap >> 37) & 0xff);
log_hex("[nvme] CAP.MPSMIN=", (cap >> 48) & 0xf);
log_hex("[nvme] CAP.MPSMAX=", (cap >> 52) & 0xf);
/* 前向声明: nvme_dump_regs 在 nvme_submit 中使用 */
static void nvme_dump_regs(const char *ctx);
/* ---- 命令提交（同步轮询, 单 outstanding） ----
* dw[16] 为完整 SQE（dw0 低 16 位不含 CID, 由本函数填入）。
* 返回 0 成功；>0 = NVMe status code；<0 超时/内部错误。
*/
static int nvme_submit(volatile u32 *sq, u32 *sq_tail, u16 sq_qid,
volatile u32 *cq, u32 *cq_head, u32 *cq_phase,
u16 cq_qid, u32 qsize, u32 *dw, u32 *out_cqe_dw0) {
dw[0] = (dw[0] & 0xffffu) | ((u32)g_cid << 16);
u32 cid = g_cid;
g_cid++;
/* 逐 dword 写入 SQE（64B），避免 freestanding 下编译器生成 memcpy 调用 */
u32 pos = *sq_tail;
for (u32 i = 0; i < 16; i++) sq[pos * 16 + i] = dw[i];
*sq_tail = (pos + 1) % qsize;
nvme_mb(); /* SQ 内容先于 doorbell 可见 */
mmio_write32(sq_tail_db(sq_qid), *sq_tail);
/* 真机超时: 使用 CAP.TO * 500ms * 10 (留 10 倍余量), 下限 5s。
* 真机 NVMe 控制器在重负载下可能需要更长的完成时间。 */
u64 submit_timeout_ms = g_timeout_ms * 10;
if (submit_timeout_ms < 5000) submit_timeout_ms = 5000;
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * submit_timeout_ms;
for (;;) {
u32 dw3 = cq[*cq_head * 4 + 3];
if (((dw3 >> 16) & 1u) == *cq_phase) {
u32 sc = (dw3 >> 17) & 0xffu;
u32 sct = (dw3 >> 25) & 0x7u;
if (out_cqe_dw0) *out_cqe_dw0 = cq[*cq_head * 4 + 0];
*cq_head = (*cq_head + 1) % qsize;
if (*cq_head == 0) *cq_phase ^= 1u;
nvme_mb();
mmio_write32(cq_head_db(cq_qid), *cq_head);
if (sc || sct) {
g_log->error("[nvme] command failed");
log_hex("[nvme] opcode=", dw[0] & 0xffu);
log_hex("[nvme] cid=", cid);
log_hex("[nvme] SC=", sc);
log_hex("[nvme] SCT=", sct);
/* SC 与 SCT 必须联合编码: SCT!=0 且 SC==0 时只返回 sc
* 会返回 0 被调用方误判为成功（CQID_INVALID 正是此类）。 */
return (int)(sc | (sct << 8));
}
return 0;
}
if (dkm_rdtsc() > deadline) {
g_log->error("[nvme] completion timeout");
log_hex("[nvme] opcode=", dw[0] & 0xffu);
nvme_dump_regs("completion timeout");
return -100;
}
__asm__ volatile("pause");
}
}
static int nvme_admin_submit(u32 *dw, u32 *out_cqe_dw0) {
return nvme_submit((volatile u32 *)g_asq.virt, &g_asq_tail, 0,
(volatile u32 *)g_acq.virt, &g_acq_head, &g_acq_phase,
0, ADMIN_QSIZE, dw, out_cqe_dw0);
}
static int nvme_io_submit(u32 *dw, u32 *out_cqe_dw0) {
return nvme_submit((volatile u32 *)g_iosq.virt, &g_iosq_tail, 1,
(volatile u32 *)g_iocq.virt, &g_iocq_head, &g_iocq_phase,
1, IO_QSIZE, dw, out_cqe_dw0);
}
/* ---- Identify ---- */
static int nvme_identify_controller(u32 *out_nn) {
u32 dw[16];
nvme_zero(dw, sizeof(dw));
nvme_zero(g_ident.virt, NVME_IDENT_BYTES);
dw[0] = NVME_ADM_IDENTIFY;
dw[1] = 0; /* NSID=0 for controller */
dw[6] = (u32)(g_ident.phys & 0xffffffffu);
dw[7] = (u32)(g_ident.phys >> 32);
dw[10] = 1; /* CNS=1: identify controller */
int rc = nvme_admin_submit(dw, 0);
if (rc != 0) return rc;
u8 *id = (u8 *)g_ident.virt;
log_str("[nvme] ctrl SN=", id + 4, 20);
log_str("[nvme] ctrl MN=", id + 24, 40);
log_str("[nvme] ctrl FR=", id + 64, 8);
/* Identify Controller 布局锚点核对（NVMe 1.4, CNS=1 数据结构）:
* byte 77 MDTS u8 最大传输 = 2^MDTS × CAP.MPSMIN 页, 0=无限制
* byte 512 SQES/CQES/MAXCMD —— QEMU 实测 0x00004466, 证明偏移基正确
* byte 516 NN u32 Number of Namespaces
* NN 核实结论 (QEMU 11.0 串口实证): NN=0x100 不是解析错位。NN 的 spec
* 语义是"控制器支持的最大合法 NSID 数"而非已挂载数量; QEMU hw/nvme
* 填的是 NVME_MAX_NAMESPACES=256 (0x100)。实际挂载的 NSID=1 由后续
* Identify Namespace(CNS=0) 成功返回确认。解析偏移 516 与 spec 一致,
* 无需修正。 */
u32 anchor512 = (u32)id[512] | ((u32)id[513] << 8) |
((u32)id[514] << 16) | ((u32)id[515] << 24);
log_hex("[nvme] ID dword@512(SQES/CQES/MAXCMD)=", anchor512);
u32 nn = (u32)id[516] | ((u32)id[517] << 8) | ((u32)id[518] << 16) | ((u32)id[519] << 24);
log_hex("[nvme] NN(max valid NSID)=", nn);
log_hex("[nvme] MDTS=", id[77]);
g_mdts = id[77];
*out_nn = nn;
return 0;
}
static int nvme_identify_namespace(u32 nsid, u64 *out_nsze, u32 *out_lbads) {
u32 dw[16];
nvme_zero(dw, sizeof(dw));
nvme_zero(g_ident.virt, NVME_IDENT_BYTES);
dw[0] = NVME_ADM_IDENTIFY;
dw[1] = nsid;
dw[6] = (u32)(g_ident.phys & 0xffffffffu);
dw[7] = (u32)(g_ident.phys >> 32);
dw[10] = 0; /* CNS=0: identify namespace */
int rc = nvme_admin_submit(dw, 0);
if (rc != 0) return rc;
u8 *id = (u8 *)g_ident.virt;
u64 nsze = 0;
for (u32 i = 0; i < 8; i++) nsze |= ((u64)id[i]) << (i * 8);
u8 flbas = id[26] & 0x0f;
u32 lbaf_off = 128 + (u32)flbas * 4;
u32 lbaf = (u32)id[lbaf_off] | ((u32)id[lbaf_off + 1] << 8) |
((u32)id[lbaf_off + 2] << 16) | ((u32)id[lbaf_off + 3] << 24);
u32 lbads = (lbaf >> 16) & 0xff;
log_hex("[nvme] NSZE=", nsze);
log_hex("[nvme] FLBAS=", flbas);
log_hex("[nvme] LBADS=", lbads);
*out_nsze = nsze;
*out_lbads = lbads;
return 0;
}
/* ---- IO queue 创建 ---- */
static int nvme_create_io_queues(void) {
u32 dw[16];
/* Create IO CQ (qid=1): PC=1, IEN=0 (轮询模式)
* CDW10 = QSIZE[31:16](0's based) | QID[15:0] —— QID 必须在低半字 */
nvme_zero(dw, sizeof(dw));
dw[0] = NVME_ADM_CREATE_IOCQ;
dw[6] = (u32)(g_iocq.phys & 0xffffffffu);
dw[7] = (u32)(g_iocq.phys >> 32);
dw[10] = ((u32)(IO_QSIZE - 1u) << 16) | 1u;
dw[11] = 1;
int rc = nvme_admin_submit(dw, 0);
if (rc != 0) {
g_log->error("[nvme] create IO CQ failed");
return rc;
}
/* Create IO SQ (qid=1, cqid=1): PC=1
* CDW10 同 Create IOCQ: QSIZE[31:16] | QID[15:0]; CDW11 = CQID[31:16] | PC */
nvme_zero(dw, sizeof(dw));
dw[0] = NVME_ADM_CREATE_IOSQ;
dw[6] = (u32)(g_iosq.phys & 0xffffffffu);
dw[7] = (u32)(g_iosq.phys >> 32);
dw[10] = ((u32)(IO_QSIZE - 1u) << 16) | 1u;
dw[11] = (1u << 16) | 1;
rc = nvme_admin_submit(dw, 0);
if (rc != 0) {
g_log->error("[nvme] create IO SQ failed");
return rc;
}
g_iosq_tail = 0;
g_iocq_head = 0;
g_iocq_phase = 1;
return 0;
}
/* ---- IO Read/Write（PRP1 + PRP2 直连 / PRP list 分档） ----
* buf_phys 必须 4KiB 对齐且物理连续（本驱动只传 g_data.phys）。
* 传输页数由 nlb × sector_size 推出，决定 PRP 布局:
* 1 页 : PRP2 = 0（单 PRP1，等价旧 bounce 路径）
* 2 页 : PRP2 = 第二页物理地址（spec 规定的两页直连形式）
* >=3 页: PRP2 = g_prp list 页物理地址, list[i] = 第 i+2 页物理地址
* 单 list 页 512 项, 上限 513 页 (2052KiB), 远大于 chunk 上限 32 页,
* 无需 list 链页。PRP list 由 CPU 写、控制器 DMA 读: x86 TSO + 编译屏障
* 保证 list 先于 doorbell 可见（与 SQE 提交路径同一约定）。
*/
static int nvme_io_rw(u8 opcode, u64 slba, u32 nlb, u64 buf_phys) {
u32 bytes = nlb * (u32)g_sector_size;
u32 pages = (bytes + 4095u) >> 12;
if (pages == 0 || (pages - 1u) > NVME_PRP_LIST_ENTRIES) return -3;
u32 dw[16];
nvme_zero(dw, sizeof(dw));
dw[0] = opcode;
dw[1] = 1; /* nsid=1 */
dw[6] = (u32)(buf_phys & 0xffffffffu);
dw[7] = (u32)(buf_phys >> 32);
if (pages == 2) {
u64 p2 = buf_phys + 4096u;
dw[8] = (u32)(p2 & 0xffffffffu);
dw[9] = (u32)(p2 >> 32);
} else if (pages >= 3) {
volatile u64 *list = (volatile u64 *)g_prp.virt;
for (u32 i = 0; i < pages - 1u; i++)
list[i] = buf_phys + 4096ULL * (u64)(i + 1u);
nvme_mb(); /* list 内容先于 doorbell 可见 */
dw[8] = (u32)(g_prp.phys & 0xffffffffu);
dw[9] = (u32)(g_prp.phys >> 32);
}
dw[10] = (u32)(slba & 0xffffffffu);
dw[11] = (u32)(slba >> 32);
dw[12] = nlb - 1u; /* NLB 0-based */
return nvme_io_submit(dw, 0);
}
/* NVM Flush (opcode 0x00): 无数据字段, 把 volatile write cache 刷到介质 */
static int nvme_io_flush(void) {
u32 dw[16];
nvme_zero(dw, sizeof(dw));
dw[0] = NVME_IO_FLUSH;
dw[1] = 1; /* nsid=1 */
return nvme_io_submit(dw, 0);
}
/* ---- block provider ----
* 调用方缓冲为任意虚拟地址（DSK 静态 BSS、vmm virtio_blk dbuf 等），
* 物理底账未知且不一定连续/对齐，不能直接进 PRP。一律 bounce 经
* g_data（物理连续 DMA 区）: 一次命令读/写满 chunk，再 memcpy 中转。
*/
static int nvme_block_read(void *ctx, u64 lba, u32 count, void *buffer) {
(void)ctx;
if (!g_io_ready || !buffer || count == 0) return -1;
u8 *out = (u8 *)buffer;
while (count > 0) {
u32 n = count > g_chunk_max_sectors ? g_chunk_max_sectors : count;
int rc = nvme_io_rw(NVME_IO_READ, lba, n, g_data.phys);
if (rc != 0) return rc;
u32 bytes = n * (u32)g_sector_size;
u8 *src = (u8 *)g_data.virt;
for (u32 i = 0; i < bytes; i++) out[i] = src[i];
out += bytes;
lba += n;
count -= n;
}
return 0;
}
static int nvme_block_write(void *ctx, u64 lba, u32 count, const void *buffer) {
(void)ctx;
if (!g_io_ready || !buffer || count == 0) return -1;
const u8 *in = (const u8 *)buffer;
while (count > 0) {
u32 n = count > g_chunk_max_sectors ? g_chunk_max_sectors : count;
u32 bytes = n * (u32)g_sector_size;
u8 *dst = (u8 *)g_data.virt;
for (u32 i = 0; i < bytes; i++) dst[i] = in[i];
/* 沿用 ahci BUG-P0-2 先例: 写前 flush CPU cache, 防 DMA 读到陈旧数据 */
__asm__ volatile("wbinvd" ::: "memory");
int rc = nvme_io_rw(NVME_IO_WRITE, lba, n, g_data.phys);
if (rc != 0) return rc;
in += bytes;
lba += n;
count -= n;
}
/* 每次写调用末尾 FLUSH 一次，保证 volatile write cache 落盘 */
return nvme_io_flush();
}
/* ---- 数据面自测（仅签名测试盘运行写测试，防实机盘数据损坏） ---- */
/* 读回 LBA0 并核对 DESHABNVME0 签名（写测试后复查无串扰用） */
static int nvme_lba0_sig_ok(void) {
static const char sig[] = "DESHABNVME0";
int rc = nvme_io_rw(NVME_IO_READ, 0, 1, g_data.phys);
if (rc != 0) return 0;
u8 *d = (u8 *)g_data.virt;
for (u32 i = 0; i < 11; i++) {
if (d[i] != (u8)sig[i]) return 0;
}
return 1;
}
/* 多页单命令读: LBA0 起 32 扇区 = 16KiB = 4 页 → PRP list 路径。
* 测试盘内容已知: LBA0 为 "DESHABNVME0" 重复填满 512B, LBA1..31 全零。 */
static void nvme_selftest_read_multipage(void) {
static u8 rbuf[16384];
static const char sig[] = "DESHABNVME0";
int rc = nvme_block_read(0, 0, 32, rbuf);
log_hex("[nvme] PRPLIST read 32sec rc=", (u64)(i64)rc);
if (rc != 0) return;
u32 bad = 0;
for (u32 i = 0; i < 512; i++) {
if (rbuf[i] != (u8)sig[i % 11]) {
bad = 1;
log_hex("[nvme] sig mismatch idx=", i);
break;
}
}
if (!bad) {
for (u32 i = 512; i < 16384; i++) {
if (rbuf[i] != 0) {
bad = 1;
log_hex("[nvme] nonzero idx=", i);
break;
}
}
}
if (bad) {
g_log->error("[nvme] PRPLIST read verify FAILED");
} else {
g_log->info("[nvme] PRPLIST read verify OK (LBA0 sig + LBA1-31 zero)");
}
}
/* 写 → FLUSH → 读回逐字节校验。sectors=16 走 PRP2 直连(2页),
* sectors=32 走 PRP list(4页)。pattern 为首 8 字节 "NVMEWRIT" + 全缓冲
* 确定式伪随机，重跑幂等（与上次写入内容一致）。 */
static void nvme_selftest_rw(u64 lba, u32 sectors) {
static u8 wpat[16384];
static u8 rbuf[16384];
u32 bytes = sectors * (u32)g_sector_size;
if (bytes > sizeof(wpat)) return;
for (u32 i = 0; i < bytes; i++) wpat[i] = (u8)(i * 31u + 17u);
wpat[0] = 'N'; wpat[1] = 'V'; wpat[2] = 'M'; wpat[3] = 'E';
wpat[4] = 'W'; wpat[5] = 'R'; wpat[6] = 'I'; wpat[7] = 'T';
log_hex("[nvme] selftest rw lba=", lba);
log_hex("[nvme] selftest rw sectors=", sectors);
int rc = nvme_block_write(0, lba, sectors, wpat);
log_hex("[nvme] write+flush rc=", (u64)(i64)rc);
if (rc != 0) return;
for (u32 i = 0; i < bytes; i++) rbuf[i] = 0;
rc = nvme_block_read(0, lba, sectors, rbuf);
log_hex("[nvme] readback rc=", (u64)(i64)rc);
if (rc != 0) return;
for (u32 i = 0; i < bytes; i++) {
if (rbuf[i] != wpat[i]) {
g_log->error("[nvme] readback MISMATCH");
log_hex("[nvme] first bad idx=", i);
log_hex("[nvme] expect=", wpat[i]);
log_hex("[nvme] actual=", rbuf[i]);
return;
}
}
g_log->info("[nvme] write+flush+readback verify OK");
}
/* ---- 真机诊断: dump 关键寄存器 ---- */
static void nvme_dump_regs(const char *ctx) {
g_log->info("[nvme] --- register dump: ");
g_log->info(ctx);
log_hex("[nvme] CAP =", mmio_read64(NVME_CAP));
log_hex("[nvme] VS =", mmio_read32(NVME_VS));
log_hex("[nvme] CC =", mmio_read32(NVME_CC));
log_hex("[nvme] CSTS =", mmio_read32(NVME_CSTS));
log_hex("[nvme] AQA =", mmio_read32(NVME_AQA));
log_hex("[nvme] ASQ =", mmio_read64(NVME_ASQ));
log_hex("[nvme] ACQ =", mmio_read64(NVME_ACQ));
/* doorbell 区域首个 entry 诊断 */
log_hex("[nvme] SQ0DB=", mmio_read32(NVME_DB_BASE));
log_hex("[nvme] CQ0DB=", mmio_read32(NVME_DB_BASE + g_db_stride));
}
/* ---- 高位 BAR MMIO 接入 ---- */
/* 真机 NVMe 控制器寄存器 + doorbell 空间可能远大于 8K。
* 典型实机: CAP.MQES 决定队列深度，doorbell 偏移 = DB_BASE + 2*QID*DSTRD*4，
* 需要映射足够大的范围覆盖至少 admin + 1 个 IO queue 对。
* 安全映射大小: 至少 64K（覆盖 QID 0-7 的 SQ/CQ doorbell）。 */
#define NVME_MAP_SIZE 0x10000
static volatile u8 *nvme_map_bar(const struct dkm_kernel_api *api, u64 bar_phys) {
u64 bar_virt = api->hhdm_offset + bar_phys;
/* 优先使用 mm_map_mmio（独立窗口映射, PCD|PWT, 不与 HHDM 空洞竞争） */
if (api->mm_map_mmio) {
void *mapped = api->mm_map_mmio(bar_phys, NVME_MAP_SIZE);
if (mapped) {
g_log->info("[nvme] BAR mapped via mm_map_mmio window");
return (volatile u8 *)mapped;
}
g_log->warn("[nvme] mm_map_mmio failed; trying HHDM fallback");
}
if (!api->mmio || !api->mmio->is_mapped) {
/* 内核无页表服务: 仅当 BAR 位于低 4G（HHDM 经验上覆盖）时冒险直用 */
if (bar_phys < 0x100000000ULL) {
g_log->warn("[nvme] no mmio api; assume HHDM covers low BAR");
return (volatile u8 *)(uintptr_t)bar_virt;
}
g_log->warn("[nvme] no mmio api and BAR above 4G; cannot access");
return (volatile u8 *)0;
}
log_hex("[nvme] CR3=", api->mmio->cr3 ? api->mmio->cr3() : 0);
/* 核实: HHDM 是否直接覆盖该 BAR（四级页表只读 walk, 无 #PF 风险） */
int level = api->mmio->is_mapped(bar_virt);
log_hex("[nvme] HHDM is_mapped(BAR)=", (u64)(i64)level);
if (level == 0) {
g_log->info("[nvme] HHDM does NOT cover BAR; mapping via page tables");
if (!api->mmio->map_mmio) return (volatile u8 *)0;
int rc = api->mmio->map_mmio(bar_phys, NVME_MAP_SIZE);
log_hex("[nvme] map_mmio rc=", (u64)(i64)rc);
if (rc != 0) return (volatile u8 *)0;
level = api->mmio->is_mapped(bar_virt);
log_hex("[nvme] post-map is_mapped=", (u64)(i64)level);
if (level == 0) {
g_log->error("[nvme] BAR still unmapped after map_mmio");
return (volatile u8 *)0;
}
} else {
g_log->info("[nvme] HHDM already covers BAR; direct access");
}
return (volatile u8 *)(uintptr_t)bar_virt;
}
/* ---- driver entry ---- */
__attribute__((visibility("default")))
int driver_init(const struct dkm_kernel_api *api,
struct dkm_driver_handle *handle) {
(void)handle;
dkm_tsc_calibrate();
if (!api || !api->log) return -1;
g_log = api->log;
dkm_instr_init(api);
g_log->info("[nvme] init begin");
u8 bus = 0;
u8 dev = 0;
u8 func = 0;
u8 bus = 0, dev = 0, func = 0;
if (nvme_find(&bus, &dev, &func) != 0) {
g_log->warn("[nvme] NVMe controller not found");
g_log->info("[nvme] driver ready");
@@ -253,89 +682,254 @@ int driver_init(const struct dkm_kernel_api *api,
}
g_log->info("[nvme] NVMe controller found");
DKM_STAT_INC(nvm0_pci_found);
log_hex("[nvme] bus=", bus);
log_hex("[nvme] dev=", dev);
log_hex("[nvme] func=", func);
u32 vd = pci_read(bus, dev, func, PCI_VENDOR_ID);
u32 vd = dkm_pci_read(bus, dev, func, PCI_VENDOR_ID);
log_hex("[nvme] vendor=", vd & 0xffff);
log_hex("[nvme] device=", vd >> 16);
u32 command = pci_read(bus, dev, func, PCI_COMMAND) & 0xffff;
command |= PCI_CMD_MEM | PCI_CMD_BUSM;
pci_write(bus, dev, func, PCI_COMMAND, command);
/* 启用 MEM + BUSM, 并禁用 INTx（轮询模式不需要中断） */
u32 command = dkm_pci_read(bus, dev, func, PCI_COMMAND) & 0xffff;
command |= PCI_CMD_MEM | PCI_CMD_BUSM | (1u << 10);
dkm_pci_write(bus, dev, func, PCI_COMMAND, command);
log_hex("[nvme] PCI command=", command);
u32 bar0_lo = pci_read(bus, dev, func, PCI_BAR0);
u32 bar1_hi = pci_read(bus, dev, func, PCI_BAR1);
u32 bar0_lo = dkm_pci_read(bus, dev, func, PCI_BAR0);
u32 bar1_hi = dkm_pci_read(bus, dev, func, PCI_BAR1);
u64 bar_phys;
if (bar0_lo & 1u) {
g_log->warn("[nvme] BAR0 is IO space; unsupported");
g_log->info("[nvme] driver ready");
return 0;
}
if ((bar0_lo & 0x6u) == 0x4u) {
bar_phys = ((u64)bar1_hi << 32) | (u64)(bar0_lo & 0xFFFFFFF0u);
} else {
bar_phys = (u64)(bar0_lo & 0xFFFFFFF0u);
}
u32 irq_line = pci_read(bus, dev, func, PCI_IRQ_LINE) & 0xff;
u32 irq_line = dkm_pci_read(bus, dev, func, PCI_IRQ_LINE) & 0xff;
log_hex("[nvme] BAR0 phys=", bar_phys);
log_hex("[nvme] PCI IRQ line=", irq_line);
if (!api->hhdm_offset || !bar_phys) {
g_log->warn("[nvme] missing HHDM or BAR0; skip MMIO enumeration");
g_log->info("[nvme] driver ready");
return 0;
}
if (bar_phys >= 0x100000000ULL) {
g_log->warn("[nvme] BAR0 is above 4G; current HHDM MMIO map cannot safely read it");
g_log->info("[nvme] discovery only; MMIO enumeration skipped");
g_log->info("[nvme] driver ready");
return 0;
}
g_regs = (volatile u8 *)(uintptr_t)(api->hhdm_offset + bar_phys);
log_hex("[nvme] HHDM offset=", api->hhdm_offset);
u64 cap = mmio_read64(NVME_CAP);
u32 vs = mmio_read32(NVME_VS);
u32 intms = mmio_read32(NVME_INTMS);
u32 intmc = mmio_read32(NVME_INTMC);
u32 cc = mmio_read32(NVME_CC);
u32 csts = mmio_read32(NVME_CSTS);
u32 nssr = mmio_read32(NVME_NSSR);
u32 aqa = mmio_read32(NVME_AQA);
u64 asq = mmio_read64(NVME_ASQ);
u64 acq = mmio_read64(NVME_ACQ);
u32 cmbloc = mmio_read32(NVME_CMBLOC);
u32 cmbsz = mmio_read32(NVME_CMBSZ);
log_hex("[nvme] CAP=", cap);
log_cap_fields(cap);
log_hex("[nvme] VS=", vs);
log_nvme_version(vs);
log_hex("[nvme] INTMS=", intms);
log_hex("[nvme] INTMC=", intmc);
log_hex("[nvme] CC=", cc);
log_hex("[nvme] CSTS=", csts);
log_hex("[nvme] NSSR=", nssr);
log_hex("[nvme] AQA=", aqa);
log_hex("[nvme] ASQ=", asq);
log_hex("[nvme] ACQ=", acq);
log_hex("[nvme] CMBLOC=", cmbloc);
log_hex("[nvme] CMBSZ=", cmbsz);
if (cc & 1u) {
g_log->warn("[nvme] controller already enabled; leaving state unchanged");
} else {
g_log->info("[nvme] controller disabled; queues not configured");
if (!api->hhdm_offset || !bar_phys) {
g_log->warn("[nvme] missing HHDM or BAR0; skip");
g_log->info("[nvme] driver ready");
return 0;
}
g_regs = nvme_map_bar(api, bar_phys);
if (!g_regs) {
g_log->error("[nvme] BAR MMIO unavailable; driver inactive");
g_log->info("[nvme] driver ready");
return 0;
}
/* ---- 寄存器枚举（验证 MMIO 可读） ---- */
u64 cap = mmio_read64(NVME_CAP);
u32 vs = mmio_read32(NVME_VS);
u32 cc = mmio_read32(NVME_CC);
u32 csts = mmio_read32(NVME_CSTS);
log_hex("[nvme] CAP=", cap);
log_hex("[nvme] VS=", vs);
log_hex("[nvme] CC=", cc);
log_hex("[nvme] CSTS=", csts);
log_hex("[nvme] CAP.MQES=", (cap & 0xffff) + 1);
log_hex("[nvme] CAP.DSTRD=", (cap >> 32) & 0xf);
log_hex("[nvme] CAP.TO=", (cap >> 24) & 0xff);
log_hex("[nvme] CAP.CSS=", (cap >> 37) & 0xff);
log_hex("[nvme] CAP.MPSMIN=", (cap >> 48) & 0xf);
log_hex("[nvme] CAP.MPSMAX=", (cap >> 52) & 0xf);
if (cap == 0 || cap == 0xFFFFFFFFFFFFFFFFULL) {
g_log->error("[nvme] CAP invalid; MMIO read broken");
g_log->info("[nvme] driver ready");
return 0;
}
if (!((cap >> 37) & 1u)) {
g_log->error("[nvme] NVM command set not supported");
g_log->info("[nvme] driver ready");
return 0;
}
g_db_stride = 4u << ((cap >> 32) & 0xf);
g_timeout_ms = ((cap >> 24) & 0xff) * 500;
if (!g_timeout_ms) g_timeout_ms = 500;
log_hex("[nvme] doorbell stride=", g_db_stride);
log_hex("[nvme] timeout ms=", g_timeout_ms);
if (!api->dma) {
g_log->warn("[nvme] DMA API unavailable; queues not configured");
g_log->info("[nvme] driver ready");
return 0;
}
/* ---- 分配队列与数据缓冲（低 4G, 4K 对齐, 清零, 物理连续） ---- */
if (api->dma->alloc_pages(1, 4096, 0x100000000ULL, &g_asq) != 0) goto dma_fail;
if (api->dma->alloc_pages(1, 4096, 0x100000000ULL, &g_acq) != 0) goto dma_fail;
if (api->dma->alloc_pages(1, 4096, 0x100000000ULL, &g_ident) != 0) goto dma_fail;
if (api->dma->alloc_pages(NVME_DATA_PAGES, 4096, 0x100000000ULL, &g_data) != 0) goto dma_fail;
if (api->dma->alloc_pages(1, 4096, 0x100000000ULL, &g_prp) != 0) goto dma_fail;
if (api->dma->alloc_pages(1, 4096, 0x100000000ULL, &g_iosq) != 0) goto dma_fail;
if (api->dma->alloc_pages(1, 4096, 0x100000000ULL, &g_iocq) != 0) goto dma_fail;
goto dma_ok;
dma_fail:
g_log->error("[nvme] DMA alloc failed");
g_log->info("[nvme] driver ready");
return 0;
dma_ok:
log_hex("[nvme] ASQ phys=", g_asq.phys);
log_hex("[nvme] ACQ phys=", g_acq.phys);
log_hex("[nvme] DATA phys=", g_data.phys);
log_hex("[nvme] PRP list phys=", g_prp.phys);
log_hex("[nvme] IOSQ phys=", g_iosq.phys);
log_hex("[nvme] IOCQ phys=", g_iocq.phys);
/* ---- 控制器复位: EN=0 等 RDY=0 ---- */
mmio_write32(NVME_INTMS, 0xFFFFFFFFu); /* 屏蔽全部中断向量 */
cc = mmio_read32(NVME_CC);
if (cc & 1u) {
mmio_write32(NVME_CC, cc & ~1u);
}
if (nvme_wait_csts(1u, 0u, g_timeout_ms) != 0) {
g_log->error("[nvme] disable failed");
nvme_dump_regs("controller disable timeout");
g_log->info("[nvme] driver ready");
return 0;
}
g_log->info("[nvme] controller disabled");
/* ---- 配置 admin queue ---- */
mmio_write32(NVME_AQA, ((u32)(ADMIN_QSIZE - 1) << 16) | (u32)(ADMIN_QSIZE - 1));
mmio_write64(NVME_ASQ, g_asq.phys);
mmio_write64(NVME_ACQ, g_acq.phys);
g_asq_tail = 0;
g_acq_head = 0;
g_acq_phase = 1;
g_cid = 1;
/* ---- 使能: EN=1, CSS=NVM, MPS=4K, AMS=RR, IOSQES=6, IOCQES=4 ---- */
mmio_write32(NVME_CC, 0x00460001u);
if (nvme_wait_csts(1u, 1u, g_timeout_ms) != 0) {
g_log->error("[nvme] enable failed");
nvme_dump_regs("controller enable timeout");
g_log->info("[nvme] driver ready");
return 0;
}
g_log->info("[nvme] controller enabled, admin queue live");
/* ---- Identify ---- */
u32 nn = 0;
if (nvme_identify_controller(&nn) != 0) {
g_log->error("[nvme] identify controller failed");
g_log->info("[nvme] driver ready");
return 0;
}
if (nn == 0) {
g_log->warn("[nvme] no namespace");
g_log->info("[nvme] driver ready");
return 0;
}
u64 nsze = 0;
u32 lbads = 0;
if (nvme_identify_namespace(1, &nsze, &lbads) != 0) {
g_log->error("[nvme] identify namespace failed");
g_log->info("[nvme] driver ready");
return 0;
}
if (lbads < 9 || lbads > 12 || nsze == 0) {
g_log->warn("[nvme] unsupported LBA format or empty NS");
log_hex("[nvme] LBADS=", lbads);
g_log->info("[nvme] driver ready");
return 0;
}
g_sector_size = 1ULL << lbads;
g_sector_count = nsze;
log_hex("[nvme] sector size=", g_sector_size);
log_hex("[nvme] sector count=", g_sector_count);
/* ---- 单命令扇区上限: min(bounce 缓冲, 2^MDTS 页, NLB 16-bit) ----
* QEMU MDTS=7 → 512KiB, bounce 32 页=128KiB 更紧, 故 chunk=256 扇区,
* 恰好覆盖 DSK FAT32 的 256 扇区批量读（一次命令读满）。 */
{
u64 buf_bytes = (u64)NVME_DATA_PAGES * 4096u;
u64 mdts_bytes = g_mdts ? ((1ULL << g_mdts) * 4096u) : buf_bytes;
u64 cap = buf_bytes < mdts_bytes ? buf_bytes : mdts_bytes;
g_chunk_max_sectors = (u32)(cap / g_sector_size);
if (g_chunk_max_sectors > 65536u) g_chunk_max_sectors = 65536u;
if (g_chunk_max_sectors == 0) g_chunk_max_sectors = 1;
log_hex("[nvme] chunk max sectors=", g_chunk_max_sectors);
}
/* ---- IO queue ---- */
if (nvme_create_io_queues() != 0) {
g_log->error("[nvme] IO queue creation failed");
g_log->info("[nvme] driver ready");
return 0;
}
g_io_ready = 1;
g_log->info("[nvme] IO queue live");
/* ---- LBA0 读取验证 ---- */
{
int rc = nvme_io_rw(NVME_IO_READ, 0, 1, g_data.phys);
log_hex("[nvme] READ LBA0 rc=", (u64)(i64)rc);
if (rc == 0) {
u8 *d = (u8 *)g_data.virt;
u32 *w = (u32 *)g_data.virt;
log_hex("[nvme] LBA0 dword0=", w[0]);
log_hex("[nvme] LBA0 dword1=", w[1]);
log_hex("[nvme] LBA0 dword2=", w[2]);
log_hex("[nvme] LBA0 dword3=", w[3]);
static const char sig[] = "DESHABNVME0";
u32 match = 1;
for (u32 i = 0; i < 11; i++) {
if (d[i] != (u8)sig[i]) { match = 0; break; }
}
if (match) {
g_log->info("[nvme] LBA0 signature OK (DESHABNVME0)");
g_test_disk = 1;
} else {
g_log->info("[nvme] LBA0 data read (no test signature)");
}
}
}
/* ---- 注册 block provider（读 + 写） ---- */
if (api->block && api->block->register_device) {
struct dkm_block_device_desc desc;
desc.name = "nvme0";
desc.sector_size = g_sector_size;
desc.sector_count = g_sector_count;
desc.ctx = 0;
desc.read = nvme_block_read;
desc.write = nvme_block_write;
int index = api->block->register_device(&desc);
log_hex("[nvme] block provider index=", (u64)(i64)index);
} else {
g_log->warn("[nvme] block API unavailable; provider not registered");
}
/* ---- 数据面自测: 写测试仅在签名测试盘上运行（防实机盘数据损坏） ---- */
if (g_test_disk) {
nvme_selftest_read_multipage(); /* 32 扇区 16KiB: PRP list 读 */
nvme_selftest_rw(1000, 16); /* 8KiB: PRP2 直连 (2 页) 写+读回 */
nvme_selftest_rw(2000, 32); /* 16KiB: PRP list (4 页) 写+读回 */
if (nvme_lba0_sig_ok()) {
g_log->info("[nvme] LBA0 sig intact after write tests");
} else {
g_log->error("[nvme] LBA0 sig CORRUPTED after write tests");
}
} else {
g_log->info("[nvme] not a test disk; write self-test skipped");
}
g_log->info("[nvme] discovery only; admin queues disabled");
g_log->info("[nvme] driver ready");
return 0;
}
CODE/DKM/pci/pci.c
+177 -1
View File
@@ -1,6 +1,11 @@
/* DKM PCI Driver — PCI bus enumeration via config space (0xCF8/0xCFC)
* Stage 0, required, depends on "irq", provides "pci".
* No IRQ needed for enumeration. Scans all buses/devices/functions.
*
* 真机增强:
* - 多总线扫描（桥接设备下游 bus > 0）
* - 网卡型号识别（常见 Intel/Realtek 网卡芯片自动标注）
* - PCI 桥检测和副总线号读取
*/
#include <stdint.h>
@@ -82,6 +87,15 @@ struct dkm_driver_desc {
#define PCI_HEADER_TYPE 0x0E
#define PCI_BAR0 0x10
/* PCI 桥配置 (header type 1) */
#define PCI_PRI_BUS 0x18
#define PCI_SEC_BUS 0x19
#define PCI_SUB_BUS 0x1A
#define PCI_CMD_IO (1u << 0)
#define PCI_CMD_MEM (1u << 1)
#define PCI_CMD_BUSM (1u << 2)
/* ---------------------------------------------------------------
* Driver descriptor
* --------------------------------------------------------------- */
@@ -94,7 +108,7 @@ const struct dkm_driver_desc driver_desc = {
.abi_version = DKM_ABI_VERSION,
.desc_size = sizeof(struct dkm_driver_desc),
.name = "pci",
.version = "0.1.0",
.version = "0.2.0",
.vendor = "Deshab",
.driver_class = 2, /* DKM_CLASS_BUS */
.stage = 0,
@@ -164,6 +178,114 @@ static void log_hex(const char *prefix, u64 val) {
g_log->info(buf);
}
/* ---- 网卡型号识别 ---- */
struct netchip_id {
u16 vendor;
u16 device;
const char *name;
u8 driver_hint; /* 0=e1000, 1=e1000e, 2=r8169, 3=igb, 4=ath9k */
};
static const struct netchip_id g_netchips[] = {
/* Intel e1000 经典 (8254x) */
{0x8086, 0x100E, "82540EM", 0},
{0x8086, 0x1004, "82543GC", 0},
{0x8086, 0x100F, "82545EM", 0},
{0x8086, 0x1010, "82546EB", 0},
{0x8086, 0x1013, "82541EI", 0},
{0x8086, 0x1019, "82547EI", 0},
{0x8086, 0x1026, "82545GM", 0},
{0x8086, 0x1075, "82547GI", 0},
{0x8086, 0x1076, "82541GI", 0},
{0x8086, 0x1079, "82546GB", 0},
{0x8086, 0x108B, "82573E", 0},
{0x8086, 0x109A, "82573L", 0},
/* Intel e1000e (8257x/I217/I218/I219) */
{0x8086, 0x10D3, "82574L", 1},
{0x8086, 0x10F5, "82578DM", 1},
{0x8086, 0x10F6, "82578DC", 1},
{0x8086, 0x1502, "82579V", 1},
{0x8086, 0x1533, "I217-V", 1},
{0x8086, 0x1534, "I217-LM", 1},
{0x8086, 0x153A, "I218-V", 1},
{0x8086, 0x153B, "I218-LM", 1},
{0x8086, 0x156F, "I219-V", 1},
{0x8086, 0x1570, "I219-LM", 1},
{0x8086, 0x157B, "I219-V-2", 1},
{0x8086, 0x15D3, "I219-V-3", 1},
{0x8086, 0x15D8, "I219-LM-3", 1},
{0x8086, 0x15BD, "I219-V-6", 1},
/* Intel I210/I211 (igb) */
{0x8086, 0x1537, "I210", 3},
{0x8086, 0x1536, "I211-AT", 3},
/* Intel 82575/82576/82580 (igb) */
{0x8086, 0x10A7, "82575EB", 3},
{0x8086, 0x10C9, "82576", 3},
{0x8086, 0x150D, "82580", 3},
{0x8086, 0x1526, "82576-2", 3},
/* Realtek RTL8111/8168/8125 (r8169) — 常见桌面/笔记本有线网卡 */
{0x10EC, 0x8167, "RTL8111/8168B", 2},
{0x10EC, 0x8168, "RTL8111/8168", 2},
{0x10EC, 0x8169, "RTL8169", 2},
{0x10EC, 0x8125, "RTL8125", 2},
{0x10EC, 0x3000, "RTL8169S", 2},
{0x10EC, 0x8136, "RTL8101E", 2},
/* Qualcomm Atheros WiFi (ath9k) */
{0x168C, 0x0023, "AR5416", 4},
{0x168C, 0x0024, "AR5418", 4},
{0x168C, 0x0027, "AR9160", 4},
{0x168C, 0x0029, "AR9280", 4},
{0x168C, 0x002A, "AR9285", 4},
{0x168C, 0x002B, "AR9287", 4},
{0x168C, 0x002D, "AR9287-2", 4},
{0x168C, 0x002E, "AR9287-3", 4},
{0x168C, 0x0030, "AR9300", 4},
{0x168C, 0x0032, "AR9485", 4},
{0x168C, 0x0034, "AR9462", 4},
{0x168C, 0x0036, "AR9485-2", 4},
{0x168C, 0x0037, "AR1111", 4},
/* Broadcom WiFi — 目前无驱动，标注识别 */
{0x14E4, 0x4315, "BCM4312-WiFi", 0xFF},
{0x14E4, 0x4365, "BCM4365-WiFi", 0xFF},
{0x14E4, 0x43A0, "BCM4360-WiFi", 0xFF},
{0x14E4, 0x43C3, "BCM4366-WiFi", 0xFF},
/* Qualcomm WiFi — 无驱动 */
{0x168C, 0x003E, "QCA6174-WiFi", 0xFF},
{0x168C, 0x0042, "QCA6390-WiFi", 0xFF},
/* Intel WiFi — 无驱动 */
{0x8086, 0x24FD, "Intel-8265-WiFi", 0xFF},
{0x8086, 0x2526, "Intel-AX200-WiFi",0xFF},
{0x8086, 0x2725, "Intel-AX210-WiFi",0xFF},
{0x8086, 0x51F0, "Intel-AX211-WiFi",0xFF},
{0x8086, 0x7A70, "Intel-BE200-WiFi",0xFF},
{0, 0, 0, 0} /* 终止符 */
};
static const char *netchip_name(u16 vendor, u16 device) {
for (int i = 0; g_netchips[i].name != 0; i++) {
if (g_netchips[i].vendor == vendor && g_netchips[i].device == device)
return g_netchips[i].name;
}
return 0;
}
static u8 netchip_driver(u16 vendor, u16 device) {
for (int i = 0; g_netchips[i].name != 0; i++) {
if (g_netchips[i].vendor == vendor && g_netchips[i].device == device)
return g_netchips[i].driver_hint;
}
return 0xFF;
}
static const char *pci_class_name(u8 base, u8 sub) {
switch (base) {
case 0x00: return "Legacy";
@@ -190,9 +312,26 @@ static const char *pci_class_name(u8 base, u8 sub) {
(void)sub;
}
/* PCI subclass names for Network class */
static const char *net_subclass_name(u8 sub) {
switch (sub) {
case 0x00: return "Ethernet";
case 0x01: return "TokenRing";
case 0x02: return "FDDI";
case 0x03: return "ATM";
case 0x04: return "ISDN";
case 0x05: return "WorldFIP";
case 0x06: return "PICMG";
case 0x80: return "Other";
default: return "Net-?";
}
}
/* ---------------------------------------------------------------
* Enumeration
* --------------------------------------------------------------- */
static u32 g_device_count;
static void pci_scan_bus(u8 bus) {
for (u8 dev = 0; dev < 32; dev++) {
u32 vid_dev = pci_cfg_read(bus, dev, 0, PCI_VENDOR_ID);
@@ -212,7 +351,9 @@ static void pci_scan_bus(u8 bus) {
u32 cc = pci_cfg_read(bus, dev, func, PCI_CLASS);
u8 class_code = (u8)(cc >> 24);
u8 subclass = (u8)((cc >> 16) & 0xff);
u8 prog_if = (u8)((cc >> 8) & 0xff);
g_device_count++;
g_log->info("[pci]");
log_hex("[pci] bus=", bus);
log_hex("[pci] dev=", dev);
@@ -223,6 +364,39 @@ static void pci_scan_bus(u8 bus) {
g_log->info(pci_class_name(class_code, subclass));
log_hex("[pci] class_hex=", class_code);
log_hex("[pci] subclass=", subclass);
/* 网卡自动识别与驱动匹配提示 */
if (class_code == 0x02) {
const char *chip_name = netchip_name(vendor2, device);
u8 drv = netchip_driver(vendor2, device);
g_log->info("[pci] NetChip");
if (chip_name) {
g_log->info(chip_name);
} else {
g_log->info("unknown");
}
g_log->info("[pci] NetSub");
g_log->info(net_subclass_name(subclass));
switch (drv) {
case 0: g_log->info("[pci] -> driver: e1000"); break;
case 1: g_log->info("[pci] -> driver: e1000 (e1000e compat)"); break;
case 2: g_log->info("[pci] -> driver: r8169 (NOT YET IMPLEMENTED)"); break;
case 3: g_log->info("[pci] -> driver: igb (e1000 compat)"); break;
case 4: g_log->info("[pci] -> driver: ath9k"); break;
case 0xFF: g_log->info("[pci] -> driver: NONE (unsupported)"); break;
default: break;
}
}
/* PCI-PCI 桥检测: 递归扫描下游总线 */
if (class_code == 0x06 && (subclass == 0x04 || subclass == 0x09)) {
u8 sec_bus = (u8)pci_cfg_read(bus, dev, func, PCI_SEC_BUS);
if (sec_bus > bus && sec_bus < 255) {
g_log->info("[pci] bridge -> secondary bus");
log_hex("[pci] sec_bus=", sec_bus);
pci_scan_bus(sec_bus);
}
}
}
}
}
@@ -238,8 +412,10 @@ int driver_init(const struct dkm_kernel_api *api,
if (!api || !api->log) return -1;
g_log = api->log;
g_device_count = 0;
g_log->info("[pci] scanning PCI bus");
pci_scan_bus(0);
log_hex("[pci] total devices=", g_device_count);
g_log->info("[pci] driver ready");
return 0;
}
CODE/DKM/ps2kbd/ps2kbd.c
+5 -36
View File
@@ -5,43 +5,10 @@
#include <stdint.h>
/* DKM ABI types */
#define DKM_DRIVER_MAGIC 0x444B4D31u
#define DKM_ABI_VERSION 1u
#include "../dkm_shared.h"
#include "../dkm_instr.h"
typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned int u32;
typedef unsigned long long u64;
typedef long long i64;
#define NULL ((void *)0)
struct dkm_log_api {
void (*info)(const char *msg);
void (*warn)(const char *msg);
void (*error)(const char *msg);
void (*panic)(const char *msg);
};
struct dkm_kernel_api {
u32 version; u32 size; u64 feature_bits;
const struct dkm_log_api *log;
const void *mem,*utsm,*irq_api,*pci_api,*dma,*vfs_api,*net,*timer,*drr;
const void *rsdp_address,*fb_address;
u64 fb_width,fb_height,fb_pitch; u16 fb_bpp;
const void *boot_modules_response;
int (*irq_register)(u8 irq, void *handler);
};
struct dkm_driver_handle;
struct dkm_driver_desc {
u32 magic; u16 abi_version; u16 desc_size;
const char *name,*version,*vendor;
u32 driver_class,stage,flags,priority;
const char *const *depends; u32 depends_count;
const char *const *provides; u32 provides_count;
u64 min_kernel_abi,feature_bits,reserved0,reserved1;
};
DKM_STAT_DECL(ps2kbd_irq_count);
/* Driver descriptor */
static const char *const g_depends[] = {"irq"};
@@ -86,6 +53,7 @@ static int kbd_handler(u8 irq) {
(void)irq;
g_last_sc = inb(0x60);
g_has_key = 1;
DKM_STAT_INC(ps2kbd_irq_count);
return 0; /* let kernel send EOI */
}
@@ -95,6 +63,7 @@ int driver_init(const struct dkm_kernel_api *api,
(void)handle;
if (!api||!api->log) return -1;
g_log=api->log;
dkm_instr_init(api);
if (!api->irq_register) {
g_log->warn("[kbd] irq_register not available");
CODE/DKM/virtio_net/virtio_net.c
+709 -42
View File
@@ -1,7 +1,19 @@
/* DKM virtio-net Driver — PCI virtio network discovery
/* DKM virtio-net Driver — virtio network device (legacy/transitional PCI)
* Stage 3 optional network driver, depends on "pci" and "irq", provides "netdev".
* This first version discovers virtio-net devices and logs PCI capabilities;
* it does not negotiate features, create virtqueues, or transmit packets.
*
* 数据路径实现（virtio spec 1.0 legacy interface，transitional device 1AF4:1001）:
* - PCI 探测 → BAR0 legacy I/O 窗口
* - reset → ACKNOWLEDGE → DRIVER → feature negotiation（MAC/STATUS 子集）
* → FEATURES_OK → virtqueue 建立（RXQ0/TXQ1, legacy split-ring 布局）
* → DRIVER_OK
* - RX: 预填 32×2048B buffer 到 RXQ avail；rx_poll 收割 used ring，
* 剥 10B virtio_net_hdr 后把以太网帧交给上层协议栈，buffer 回收重填
* - TX: 8×2048B buffer 池（desc 索引与 buffer 1:1 映射），hdr+帧单描述符，
* QueueNotify kick，与 e1000 一致的同步等待完成（TSC 超时兜底）
* - boot 自测: 经真实 QEMU user-net 后端向 10.0.2.2 DHCP server 发 DISCOVER、
* 收 OFFER，一次跑通证明 TX/RX 全链路（DMA/notify/used ring）
* - IRQ 按 IDT vector（irq_line+0x20）注册做 ISR 统计；PIC 未解屏蔽时
* 自动退化为纯轮询，不依赖中断正确性
*/
#include "../dkm_shared.h"
@@ -10,11 +22,7 @@
#define PCI_CLASS_REG 0x08 /* 32-bit class/subclass/prog_if/revision */
#define PCI_VENDOR_VIRTIO 0x1AF4u
#define VIRTIO_TRANS_DEVICE_MIN 0x1000u
#define VIRTIO_TRANS_DEVICE_MAX 0x103Fu
#define VIRTIO_MODERN_DEVICE_MIN 0x1040u
#define VIRTIO_MODERN_DEVICE_MAX 0x107Fu
#define VIRTIO_NET_DEVICE_ID 1u
/* PCI device ID 约定见 virtio_is_net_device(): transitional net=0x1000, modern net=0x1041 */
#define VIRTIO_PCI_CAP_COMMON_CFG 1u
#define VIRTIO_PCI_CAP_NOTIFY_CFG 2u
@@ -22,6 +30,76 @@
#define VIRTIO_PCI_CAP_DEVICE_CFG 4u
#define VIRTIO_PCI_CAP_PCI_CFG 5u
/* ---- legacy I/O 寄存器偏移（virtio spec 1.0 §4.1.4, BAR0 I/O space） ---- */
#define VP_HOST_FEATURES 0x00u /* u32 RO: device feature bits 0..31 */
#define VP_GUEST_FEATURES 0x04u /* u32 WO: driver 接受的 feature 子集 */
#define VP_QUEUE_PFN 0x08u /* u32 WO: 队列物理地址 >> 12 */
#define VP_QUEUE_NUM 0x0Cu /* u16 RO: 队列容量（2 的幂） */
#define VP_QUEUE_SEL 0x0Eu /* u16 WO: 选择操作的队列 */
#define VP_QUEUE_NOTIFY 0x10u /* u16 WO: kick 指定队列 */
#define VP_STATUS 0x12u /* u8 RW: device status */
#define VP_ISR 0x13u /* u8 RO: ISR，读即清除 */
#define VP_NET_CFG_MAC 0x14u /* 6B RO: MAC（device config 区起始） */
/* ---- device status bits ---- */
#define VS_ACKNOWLEDGE 0x01u
#define VS_DRIVER 0x02u
#define VS_DRIVER_OK 0x04u
#define VS_FEATURES_OK 0x08u
/* ---- virtio-net feature bits（spec §5.1.6），只取最小子集 ---- */
#define VNET_F_MAC (1u << 5) /* device config 提供 MAC */
#define VNET_F_STATUS (1u << 16) /* device config 提供 link status */
/* ---- vring ---- */
#define VRING_DESC_F_WRITE 2u /* descriptor 由 device 写（RX） */
#define VNET_HDR_SIZE 10u /* 未协商 MRG_RXBUF → virtio_net_hdr 为 10 字节 */
#define RX_BUF_COUNT 32u /* RX 预填 buffer 数（不必填满整个队列） */
#define TX_BUF_COUNT 8u /* TX buffer 池大小 */
#define VNET_BUF_SIZE 2048u /* 10B hdr + 最大 1518B 以太网帧 */
/* split-ring 结构（legacy，x86 上字段为 guest 原生小端序） */
struct vring_desc {
u64 addr;
u32 len;
u16 flags;
u16 next;
};
struct vring_used_elem {
u32 id;
u32 len;
};
struct vring_avail {
u16 flags;
u16 idx;
u16 ring[];
};
struct vring_used {
u16 flags;
u16 idx;
struct vring_used_elem ring[];
};
/* 单个 virtqueue 的运行时视图。
* legacy 布局（spec §2.4.2）:
* base+0 : descriptor table（16B × size）
* base+16*size : avail ring（4 + 2×size + 2B used_event）
* align4K(16*size + 6 + 2*size): used ring（4 + 8×size + 2B avail_event）
* 整段必须物理连续且 4KiB 对齐（QueuePFN 只接受页帧号）。 */
struct virtq {
u16 qsel; /* queue select 索引（0=RX, 1=TX） */
u16 size; /* device 报告的队列容量 */
u64 phys; /* 队列内存物理基址（页对齐） */
struct vring_desc *desc;
struct vring_avail *avail;
struct vring_used *used;
u16 last_used_idx; /* driver 侧 used ring 收割游标 */
};
static const char *const g_depends[] = { "pci", "irq" };
static const char *const g_provides[] = { "netdev" };
@@ -31,7 +109,7 @@ const struct dkm_driver_desc driver_desc = {
.abi_version = DKM_ABI_VERSION,
.desc_size = sizeof(struct dkm_driver_desc),
.name = "virtio_net",
.version = "0.1.0",
.version = "0.2.0",
.vendor = "Deshab",
.driver_class = 9, /* DKM_CLASS_NET */
.stage = 3,
@@ -49,6 +127,48 @@ const struct dkm_driver_desc driver_desc = {
static const struct dkm_log_api *g_log;
/* ---- 全局设备状态 ---- */
static u16 g_io_base; /* legacy I/O BAR 基址，0=未初始化 */
static struct virtq g_rxq;
static struct virtq g_txq;
static u8 *g_rx_buf[RX_BUF_COUNT];
static u64 g_rx_buf_phys[RX_BUF_COUNT];
static u8 *g_tx_buf[TX_BUF_COUNT];
static u64 g_tx_buf_phys[TX_BUF_COUNT];
static u8 g_tx_free[TX_BUF_COUNT]; /* 1=空闲可提交 */
static int g_ready; /* 数据路径就绪标志 */
static u8 g_mac[6];
/* ---- 统计计数器（串口可观测） ---- */
static u64 g_stat_tx_ok;
static u64 g_stat_tx_err;
static u64 g_stat_tx_timeout;
static u64 g_stat_rx_ok;
static u64 g_stat_rx_err;
static u64 g_stat_rx_overflow;
static u64 g_stat_irq_count;
/* ---- 自测缓冲区（静态 bss，避免内核栈压力） ---- */
static u8 g_st_frame[360]; /* DHCP DISCOVER 帧（342B） */
static u8 g_st_buf[2048]; /* RX 收割暂存 */
/* ---- 16-bit I/O port 访问（dkm_shared.h 只提供 8/32 位） ---- */
static __inline__ void vn_outw(u16 port, u16 value) {
__asm__ volatile ("outw %0, %1" :: "a"(value), "Nd"(port));
}
static __inline__ u16 vn_inw(u16 port) {
u16 value;
__asm__ volatile ("inw %1, %0" : "=a"(value) : "Nd"(port));
return value;
}
/* device DMA 内存屏障：描述符/ring 写入必须先于 idx 更新对设备可见。
* x86 store-store 有序，compiler fence + mfence 双保险（控制路径，开销可忽略）。 */
static __inline__ void vn_fence(void) {
__asm__ volatile("mfence" ::: "memory");
}
static u32 pci_read(u8 bus, u8 dev, u8 func, u8 reg) {
return dkm_pci_read(bus, dev, func, reg);
}
@@ -61,6 +181,17 @@ static u8 pci_read8(u8 bus, u8 dev, u8 func, u8 reg) {
return dkm_pci_read8(bus, dev, func, reg);
}
static void copy_bytes(void *dst, const void *src, u32 len) {
u8 *d = (u8 *)dst;
const u8 *s = (const u8 *)src;
for (u32 i = 0; i < len; i++) d[i] = s[i];
}
static void zero_bytes(void *dst, u32 len) {
u8 *d = (u8 *)dst;
for (u32 i = 0; i < len; i++) d[i] = 0;
}
static void log_hex(const char *prefix, u64 value) {
static const char hex[] = "0123456789abcdef";
char buf[19];
@@ -75,14 +206,51 @@ static void log_hex(const char *prefix, u64 value) {
g_log->info(buf);
}
static u16 virtio_subsystem_id(u16 device_id) {
if (device_id >= VIRTIO_TRANS_DEVICE_MIN && device_id <= VIRTIO_TRANS_DEVICE_MAX) {
return (u16)(device_id - VIRTIO_TRANS_DEVICE_MIN);
static u32 dec_u32(u32 v, char *out) {
char tmp[10];
u32 n = 0;
if (v == 0) { out[0] = '0'; return 1; }
while (v && n < 10) { tmp[n++] = (char)('0' + (v % 10)); v /= 10; }
for (u32 i = 0; i < n; i++) out[i] = tmp[n - 1 - i];
return n;
}
/* 打印 IPv4 地址。ip 为主机序（高字节 = 第一个 octet，如 0x0A00020F = 10.0.2.15）。 */
static void log_ipv4(const char *prefix, u32 ip) {
char buf[24];
u32 p = 0;
for (u32 i = 0; i < 4; i++) {
u8 octet = (u8)(ip >> ((3 - i) * 8));
p += dec_u32(octet, buf + p);
if (i < 3) buf[p++] = '.';
}
if (device_id >= VIRTIO_MODERN_DEVICE_MIN && device_id <= VIRTIO_MODERN_DEVICE_MAX) {
return (u16)(device_id - VIRTIO_MODERN_DEVICE_MIN);
buf[p] = 0;
g_log->info(prefix);
g_log->info(buf);
}
static void log_mac(const char *prefix, const u8 mac[6]) {
static const char hex[] = "0123456789abcdef";
char buf[18];
u32 p = 0;
for (u32 i = 0; i < 6; i++) {
buf[p++] = hex[(mac[i] >> 4) & 0xf];
buf[p++] = hex[mac[i] & 0xf];
if (i < 5) buf[p++] = ':';
}
return 0xffff;
buf[p] = 0;
g_log->info(prefix);
g_log->info(buf);
}
/* virtio-net PCI device ID 判定。
* QEMU 编号约定（virtio 1.0 spec §4.1.2.1 + QEMU hw/virtio/virtio-pci.c）:
* - transitional/legacy: 0x1000=net, 0x1001=blk, 0x1002=balloon, ...
* （历史 table，不是 0x1000+virtio_id；virtio device id 由 PCI Subsystem ID 携带）
* - modern-only: 0x1040+virtio_id → net = 0x1041
* class code (0x02=Network) 作为第二判据，与 virtio-blk(0x1001/Storage) 区分。 */
static int virtio_is_net_device(u16 device_id) {
return (device_id == 0x1000u || device_id == 0x1041u) ? 1 : 0;
}
static int virtio_net_find(u8 *out_bus, u8 *out_dev, u8 *out_func) {
@@ -99,10 +267,9 @@ static int virtio_net_find(u8 *out_bus, u8 *out_dev, u8 *out_func) {
u16 device = (u16)(vd2 >> 16);
if (vendor != PCI_VENDOR_VIRTIO) continue;
u16 virtio_id = virtio_subsystem_id(device);
u32 class_reg = pci_read(bus, dev, func, PCI_CLASS_REG);
u8 class_code = (u8)(class_reg >> 24);
if (virtio_id == VIRTIO_NET_DEVICE_ID && class_code == 0x02) {
if (virtio_is_net_device(device) && class_code == 0x02) {
*out_bus = bus;
*out_dev = dev;
*out_func = func;
@@ -178,28 +345,455 @@ static void log_virtio_caps(u8 bus, u8 dev, u8 func) {
}
}
/* ---- tx/rx_poll stub: 安全占位, 不实际收发, 防止 netdev 调用 NULL ---- */
static int virtio_net_tx(void *ctx, const void *packet, u32 length) {
(void)ctx;
(void)packet;
/* 仅记录日志, 避免每次发包刷屏: 仅记录长度 */
static u32 s_tx_log_guard = 0;
if (s_tx_log_guard < 4) {
g_log->info("[virtio_net] tx stub: packet dropped (virtqueues not enabled)");
log_hex("[virtio_net] tx stub len=", length);
s_tx_log_guard++;
/* ================= legacy device 初始化序列 ================= */
/* device status 寄存器读写（8 位） */
static void vdev_set_status(u8 status) {
dkm_outb((u16)(g_io_base + VP_STATUS), status);
}
static u8 vdev_get_status(void) {
return dkm_inb((u16)(g_io_base + VP_STATUS));
}
/* reset → ACKNOWLEDGE → DRIVER → feature 协商 → FEATURES_OK。
* 返回 0 成功；失败返回负数（调用方回退 stub 注册）。 */
static int vdev_negotiate(void) {
/* 1. reset: 写 0 并等待设备读回 0（spec §4.1.4.3 要求轮询确认） */
vdev_set_status(0);
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 100;
while (vdev_get_status() != 0) {
if (dkm_rdtsc() > deadline) {
g_log->error("[virtio_net] device reset timeout");
return -1;
}
__asm__ volatile("pause");
}
/* 2/3. ACKNOWLEDGE → DRIVER（status 位按序累积写入） */
vdev_set_status(VS_ACKNOWLEDGE);
vdev_set_status(VS_ACKNOWLEDGE | VS_DRIVER);
/* 4. feature negotiation: 只取 MAC + STATUS 子集。
* 不接受 MRG_RXBUF/CTRL_VQ/CSUM/GSO 等复杂特性 →
* virtio_net_hdr 固定 10 字节、无控制队列、无卸载，数据路径最简。 */
u32 dev_feat = dkm_inl((u16)(g_io_base + VP_HOST_FEATURES));
log_hex("[virtio_net] device features=", dev_feat);
u32 subset = dev_feat & (VNET_F_MAC | VNET_F_STATUS);
dkm_outl((u16)(g_io_base + VP_GUEST_FEATURES), subset);
log_hex("[virtio_net] guest features=", subset);
/* 5. FEATURES_OK 并回读确认（spec §3.1.1）。
* 纯 legacy（pre-1.0）设备没有该位概念；QEMU transitional 实现 1.0 语义会置位。
* 未置位时按 legacy 兼容策略 warn 后继续（与 Linux legacy 驱动一致）。 */
vdev_set_status(VS_ACKNOWLEDGE | VS_DRIVER | VS_FEATURES_OK);
u8 st = vdev_get_status();
log_hex("[virtio_net] status after FEATURES_OK=", st);
if (!(st & VS_FEATURES_OK)) {
g_log->warn("[virtio_net] FEATURES_OK not echoed; continuing as pre-1.0 legacy device");
} else {
g_log->info("[virtio_net] FEATURES_OK confirmed");
}
return 0;
}
/* QueuePFN = 物理地址 >> 12（legacy 队列地址寄存器语义） */
static void vn_outl_pfn(const struct virtq *q) {
dkm_outl((u16)(g_io_base + VP_QUEUE_PFN), (u32)(q->phys >> 12));
}
/* 建立单个 virtqueue：读容量 → 计算 legacy 布局 → 分配连续页 → 写 QueuePFN。 */
static int vq_setup(const struct dkm_dma_api *dma, struct virtq *q, u16 qsel) {
q->qsel = qsel;
q->last_used_idx = 0;
vn_outw((u16)(g_io_base + VP_QUEUE_SEL), qsel);
q->size = vn_inw((u16)(g_io_base + VP_QUEUE_NUM));
if (q->size == 0) {
g_log->error("[virtio_net] queue size is 0");
return -1;
}
/* legacy 布局（spec §2.4.2）:
* desc @ 0，avail @ 16*size，used @ 4KiB 对齐边界，整段物理连续。 */
u64 avail_off = 16ULL * q->size;
u64 used_off = (avail_off + 6ULL + 2ULL * q->size + 4095ULL) & ~4095ULL;
u64 total = used_off + 6ULL + 8ULL * q->size;
u64 pages = (total + 4095ULL) / 4096ULL;
struct dkm_dma_buffer buf;
/* 低 4G + 页对齐：QueuePFN 只接受 phys>>12，alloc_pages 返回已清零页 */
if (dma->alloc_pages(pages, 4096, 0x100000000ULL, &buf) != 0) {
g_log->error("[virtio_net] queue memory alloc failed");
return -2;
}
q->phys = buf.phys;
q->desc = (struct vring_desc *)buf.virt;
q->avail = (struct vring_avail *)((u8 *)buf.virt + avail_off);
q->used = (struct vring_used *)((u8 *)buf.virt + used_off);
vn_outl_pfn(q);
return 0;
}
/* kick 队列：写 queue index 到 QueueNotify */
static void vq_notify(const struct virtq *q) {
vn_outw((u16)(g_io_base + VP_QUEUE_NOTIFY), q->qsel);
}
/* 把描述符挂到 avail ring 并推进 idx。
* 顺序约束：desc 字段必须先写好（调用方责任），ring 写入 → fence → idx 推进。 */
static void vq_push_avail(struct virtq *q, u16 desc_idx) {
u16 slot = (u16)(q->avail->idx % q->size);
q->avail->ring[slot] = desc_idx;
vn_fence();
q->avail->idx = (u16)(q->avail->idx + 1u);
vn_fence();
}
/* 从 used ring 取一个完成项。返回 0=取到，1=空。 */
static int vq_pop_used(struct virtq *q, u16 *out_id, u32 *out_len) {
vn_fence();
if (q->used->idx == q->last_used_idx) return 1;
struct vring_used_elem *e = &q->used->ring[q->last_used_idx % q->size];
*out_id = (u16)e->id;
*out_len = e->len;
vn_fence();
q->last_used_idx = (u16)(q->last_used_idx + 1u);
return 0;
}
/* ================= RX 路径 ================= */
/* 回收 RX buffer：重置描述符并重新挂回 avail ring，保持 RX 水位。 */
static void rx_requeue(u16 id) {
g_rxq.desc[id].addr = g_rx_buf_phys[id];
g_rxq.desc[id].len = VNET_BUF_SIZE;
g_rxq.desc[id].flags = VRING_DESC_F_WRITE;
g_rxq.desc[id].next = 0;
vq_push_avail(&g_rxq, id);
vq_notify(&g_rxq);
}
/* 预填全部 RX buffer（单描述符覆盖 hdr+帧，device 可写）。 */
static int rx_prefill(const struct dkm_dma_api *dma) {
for (u32 i = 0; i < RX_BUF_COUNT; i++) {
struct dkm_dma_buffer b;
if (dma->alloc_pages(1, 4096, 0x100000000ULL, &b) != 0) {
g_log->error("[virtio_net] rx buffer alloc failed");
log_hex("[virtio_net] filled=", i);
return -1;
}
g_rx_buf[i] = (u8 *)b.virt;
g_rx_buf_phys[i] = b.phys;
g_rxq.desc[i].addr = b.phys;
g_rxq.desc[i].len = VNET_BUF_SIZE;
g_rxq.desc[i].flags = VRING_DESC_F_WRITE;
g_rxq.desc[i].next = 0;
g_rxq.avail->ring[i] = (u16)i;
}
vn_fence();
g_rxq.avail->idx = (u16)RX_BUF_COUNT;
vn_fence();
log_hex("[virtio_net] rx buffers prefilled=", RX_BUF_COUNT);
return 0;
}
/* netdev rx_poll 回调（契约与 e1000 一致）:
* 返回 0 且 *out_length>0 = 收到一帧；1 = 无数据；负数 = 错误。 */
static int virtio_net_rx_poll(void *ctx, void *buffer, u32 capacity, u32 *out_length) {
(void)ctx;
if (out_length) *out_length = 0;
if (!g_ready || !buffer || !out_length) return -1;
u16 id;
u32 total;
if (vq_pop_used(&g_rxq, &id, &total) != 0) return 1; /* 无完成项 */
/* 防御：id 越界或总长小于 hdr → 丢帧（id 无效时无法回收，只计数） */
if (id >= RX_BUF_COUNT) {
g_stat_rx_err++;
return -3;
}
if (total < VNET_HDR_SIZE) {
g_stat_rx_err++;
rx_requeue(id);
return -3;
}
u32 frame_len = total - VNET_HDR_SIZE; /* 剥掉 virtio_net_hdr */
if (frame_len > capacity) {
g_stat_rx_overflow++;
rx_requeue(id);
return -2;
}
copy_bytes(buffer, g_rx_buf[id] + VNET_HDR_SIZE, frame_len);
*out_length = frame_len;
rx_requeue(id);
g_stat_rx_ok++;
return 0;
}
/* ================= TX 路径 ================= */
/* 收割 TX used ring，把完成的 buffer 标回空闲。 */
static void tx_reclaim(void) {
u16 id;
u32 len;
while (vq_pop_used(&g_txq, &id, &len) == 0) {
(void)len; /* TX 的 used.len 无语义，仅 id 用于回收 */
if (id < TX_BUF_COUNT) g_tx_free[id] = 1;
}
}
static int tx_buffers_alloc(const struct dkm_dma_api *dma) {
for (u32 i = 0; i < TX_BUF_COUNT; i++) {
struct dkm_dma_buffer b;
if (dma->alloc_pages(1, 4096, 0x100000000ULL, &b) != 0) {
g_log->error("[virtio_net] tx buffer alloc failed");
return -1;
}
g_tx_buf[i] = (u8 *)b.virt;
g_tx_buf_phys[i] = b.phys;
g_tx_free[i] = 1;
}
return 0;
}
/* netdev tx 回调：hdr(10B 全零，无卸载) + 帧单描述符提交，kick 后同步等待完成。
* 与 e1000 行为一致：TSC 1 秒超时兜底，避免上层协议栈无限阻塞。 */
static int virtio_net_tx(void *ctx, const void *packet, u32 length) {
(void)ctx;
if (!g_ready || !packet || length == 0 || length + VNET_HDR_SIZE > VNET_BUF_SIZE) {
g_stat_tx_err++;
return -1;
}
tx_reclaim();
int slot = -1;
for (u32 i = 0; i < TX_BUF_COUNT; i++) {
if (g_tx_free[i]) { slot = (int)i; break; }
}
if (slot < 0) {
g_stat_tx_err++;
g_log->warn("[virtio_net] tx ring full");
return -2;
}
u8 *buf = g_tx_buf[slot];
zero_bytes(buf, VNET_HDR_SIZE); /* virtio_net_hdr: flags/gso 全 0 */
copy_bytes(buf + VNET_HDR_SIZE, packet, length);
/* desc 索引与 buffer 1:1 映射，回收时 used.id 即 buffer 索引 */
g_txq.desc[slot].addr = g_tx_buf_phys[slot];
g_txq.desc[slot].len = length + VNET_HDR_SIZE;
g_txq.desc[slot].flags = 0; /* device 只读 */
g_txq.desc[slot].next = 0;
g_tx_free[slot] = 0;
vq_push_avail(&g_txq, (u16)slot);
vq_notify(&g_txq);
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 1000;
while (dkm_rdtsc() < deadline) {
tx_reclaim();
if (g_tx_free[slot]) {
g_stat_tx_ok++;
return 0;
}
__asm__ volatile("pause");
}
g_stat_tx_timeout++;
g_log->warn("[virtio_net] tx timeout");
log_hex("[virtio_net] txq used idx=", g_txq.used->idx);
log_hex("[virtio_net] txq last_used=", g_txq.last_used_idx);
return -3;
}
/* ================= IRQ（辅助统计，数据路径不依赖） ================= */
static int virtio_net_irq(u8 irq) {
(void)irq;
if (!g_io_base) return 0;
/* ISR 读即清除；bit0=queue used buffer 通知，bit1=config change */
u8 isr = dkm_inb((u16)(g_io_base + VP_ISR));
if (isr & 0x1u) g_stat_irq_count++;
return 0; /* 不 suppress EOI，由 IDT 发 PIC EOI */
}
/* ================= boot 自测：DHCP DISCOVER/OFFER ================= */
#define ST_XID 0x4B1D3A5Fu
#define ST_FRAME_LEN 342u /* 14 eth + 20 ip + 8 udp + 300 bootp */
/* 构造 DHCP DISCOVER（全手工大端字段，避免 packed struct 字节序歧义）。
* BOOTP 载荷补齐到 300 字节（RFC 1542 最小值，slirp DHCP server 容忍度最高）。 */
static void st_build_discover(const u8 mac[6]) {
u8 *f = g_st_frame;
zero_bytes(f, ST_FRAME_LEN);
/* Ethernet: 广播 DISCOVER */
for (u32 i = 0; i < 6; i++) f[i] = 0xff;
for (u32 i = 0; i < 6; i++) f[6 + i] = mac[i];
f[12] = 0x08; f[13] = 0x00; /* IPv4 */
/* IPv4 @14 */
u8 *ip = f + 14;
ip[0] = 0x45; /* v4, ihl=5 */
ip[1] = 0x00;
ip[2] = 0x01; ip[3] = 0x48; /* total len = 328 */
ip[4] = 0x00; ip[5] = 0x00; /* id */
ip[6] = 0x00; ip[7] = 0x00; /* flags/frag */
ip[8] = 64; /* ttl */
ip[9] = 17; /* UDP */
ip[10] = 0x00; ip[11] = 0x00; /* cksum 占位 */
/* src 0.0.0.0 @12..15 保持 0 */
ip[16] = 0xff; ip[17] = 0xff; ip[18] = 0xff; ip[19] = 0xff;
/* IP header checksum（标准 1's complement，大端语义直接按字节对累加） */
u32 sum = 0;
for (u32 i = 0; i < 20; i += 2) sum += ((u32)ip[i] << 8) | ip[i + 1];
while (sum >> 16) sum = (sum & 0xffffu) + (sum >> 16);
u16 ck = (u16)(~sum & 0xffffu);
ip[10] = (u8)(ck >> 8); ip[11] = (u8)ck;
/* UDP @34: 68 → 67，校验和置 0（IPv4 下合法） */
u8 *udp = f + 34;
udp[0] = 0x00; udp[1] = 68;
udp[2] = 0x00; udp[3] = 67;
udp[4] = 0x01; udp[5] = 0x34; /* len = 308 */
udp[6] = 0x00; udp[7] = 0x00;
/* BOOTP @42（固定 236B + magic 4B + options，pad 到 300B） */
u8 *bp = f + 42;
bp[0] = 1; /* op = BOOTREQUEST */
bp[1] = 1; /* htype = Ethernet */
bp[2] = 6; /* hlen */
bp[3] = 0; /* hops */
bp[4] = (u8)(ST_XID >> 24);
bp[5] = (u8)(ST_XID >> 16);
bp[6] = (u8)(ST_XID >> 8);
bp[7] = (u8)ST_XID;
/* secs @8..9 = 0 */
bp[10] = 0x80; bp[11] = 0x00; /* flags = broadcast */
/* ciaddr/yiaddr/siaddr/giaddr @12..27 = 0 */
for (u32 i = 0; i < 6; i++) bp[28 + i] = mac[i]; /* chaddr（16B，后 10B 为 0） */
/* sname @44..107, file @108..235 = 0 */
bp[236] = 0x63; bp[237] = 0x82; bp[238] = 0x53; bp[239] = 0x63; /* magic cookie */
u8 *opt = bp + 240;
opt[0] = 53; opt[1] = 1; opt[2] = 1; /* DHCP msg type = DISCOVER */
opt[3] = 55; opt[4] = 4; /* param request list */
opt[5] = 1; opt[6] = 3; opt[7] = 6; opt[8] = 15; /* subnet/router/dns/domain */
opt[9] = 255; /* end */
}
/* 解析帧是否为匹配 XID 的 DHCP OFFER。
* 返回 2=OFFER（*out_yiaddr 有效，主机序），5=ACK，0=无关帧。 */
static int st_parse_reply(const u8 *f, u32 len, u32 *out_yiaddr) {
if (len < 14 + 20 + 8 + 240) return 0;
if (f[12] != 0x08 || f[13] != 0x00) return 0; /* 非 IPv4 */
u32 ihl = (u32)(f[14] & 0x0f) * 4u;
if (ihl < 20) return 0;
if (f[14 + 9] != 17) return 0; /* 非 UDP */
u32 udp = 14 + ihl;
if (len < udp + 8 + 240) return 0;
u16 dport = (u16)(((u16)f[udp + 2] << 8) | f[udp + 3]);
if (dport != 68) return 0; /* 非 DHCP 回包 */
u32 bp = udp + 8;
if (f[bp] != 2) return 0; /* 非 BOOTREPLY */
u32 xid = ((u32)f[bp + 4] << 24) | ((u32)f[bp + 5] << 16) |
((u32)f[bp + 6] << 8) | f[bp + 7];
if (xid != ST_XID) return 0; /* XID 不匹配 */
if (f[bp + 236] != 0x63 || f[bp + 237] != 0x82 ||
f[bp + 238] != 0x53 || f[bp + 239] != 0x63) return 0;
u32 yiaddr = ((u32)f[bp + 16] << 24) | ((u32)f[bp + 17] << 16) |
((u32)f[bp + 18] << 8) | f[bp + 19];
/* options 区扫描 msg type（53） */
u32 i = bp + 240;
while (i < len) {
u8 o = f[i];
if (o == 255) break;
if (o == 0) { i++; continue; }
if (i + 1 >= len) break;
u8 olen = f[i + 1];
if (o == 53 && olen == 1 && i + 2 < len) {
u8 mtype = f[i + 2];
if (mtype == 2 || mtype == 5) {
*out_yiaddr = yiaddr;
return mtype;
}
return 0;
}
i += 2u + olen;
}
return 0;
}
/* boot 自测：经真实网络后端验证 TX（desc+notify+used 回收）与
* RX（avail 预填 + used 收割 + hdr 剥离）全链路。
* 任何失败只记日志，绝不阻塞/崩溃启动流程。 */
static void virtio_net_selftest(void) {
if (!g_ready) return;
g_log->info("[virtio_net] selftest begin: DHCP DISCOVER -> expect OFFER from backend");
st_build_discover(g_mac);
int txrc = virtio_net_tx(0, g_st_frame, ST_FRAME_LEN);
log_hex("[virtio_net] selftest tx rc=", (u64)(i64)txrc);
if (txrc != 0) {
g_log->warn("[virtio_net] selftest: TX path FAILED");
return;
}
g_log->info("[virtio_net] selftest: TX path ok (descriptor consumed by device)");
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 2000;
u32 frames = 0;
while (dkm_rdtsc() < deadline) {
u32 flen = 0;
int rc = virtio_net_rx_poll(0, g_st_buf, sizeof(g_st_buf), &flen);
if (rc == 0 && flen > 0) {
frames++;
u32 ethertype = ((u32)g_st_buf[12] << 8) | g_st_buf[13];
log_hex("[virtio_net] selftest rx frame len=", flen);
log_hex("[virtio_net] ethertype=", ethertype);
u32 yiaddr = 0;
int t = st_parse_reply(g_st_buf, flen, &yiaddr);
if (t == 2) {
log_ipv4("[virtio_net] selftest: DHCP OFFER yiaddr=", yiaddr);
g_log->info("[virtio_net] selftest PASS: RX path ok (frame from real backend)");
return;
}
}
__asm__ volatile("pause");
}
g_log->warn("[virtio_net] selftest: no DHCP OFFER within 2000ms");
log_hex("[virtio_net] selftest rx frames seen=", frames);
}
/* ================= stub（初始化失败时的安全占位，与历史行为一致） ================= */
static int virtio_net_tx_stub(void *ctx, const void *packet, u32 length) {
(void)ctx;
(void)packet;
log_hex("[virtio_net] tx stub: packet dropped, len=", length);
return 0;
}
static int virtio_net_rx_poll_stub(void *ctx, void *buffer, u32 capacity, u32 *out_length) {
(void)ctx;
(void)buffer;
(void)capacity;
if (out_length) *out_length = 0;
return 0; /* 无数据, virtqueues 未启用 */
return 0;
}
/* ================= driver 入口 ================= */
__attribute__((visibility("default")))
int driver_init(const struct dkm_kernel_api *api,
struct dkm_driver_handle *handle) {
@@ -230,19 +824,11 @@ int driver_init(const struct dkm_kernel_api *api,
u32 vd = pci_read(bus, dev, func, PCI_VENDOR_ID);
u16 vendor = (u16)(vd & 0xffff);
u16 device = (u16)(vd >> 16);
u16 virtio_id = virtio_subsystem_id(device);
u32 class_reg = pci_read(bus, dev, func, PCI_CLASS_REG);
u8 revision = (u8)(class_reg & 0xff);
u8 prog_if = (u8)((class_reg >> 8) & 0xff);
u8 subclass = (u8)((class_reg >> 16) & 0xff);
u8 class_code = (u8)((class_reg >> 24) & 0xff);
log_hex("[virtio_net] vendor=", vendor);
log_hex("[virtio_net] device=", device);
log_hex("[virtio_net] virtio_id=", virtio_id);
log_hex("[virtio_net] class=", class_code);
log_hex("[virtio_net] subclass=", subclass);
log_hex("[virtio_net] prog_if=", prog_if);
log_hex("[virtio_net] revision=", revision);
u32 command = pci_read(bus, dev, func, PCI_COMMAND) & 0xffff;
@@ -256,21 +842,96 @@ int driver_init(const struct dkm_kernel_api *api,
log_bars(bus, dev, func);
log_virtio_caps(bus, dev, func);
/* ---- legacy I/O BAR 检查：transitional 设备 BAR0 必须是 I/O space ---- */
u32 bar0 = pci_read(bus, dev, func, PCI_BAR0);
int datapath_ok = 0;
if (!(bar0 & 1u)) {
g_log->warn("[virtio_net] BAR0 is not I/O space (modern-only device?); legacy path unavailable");
} else if (!api->dma || !api->dma->alloc_pages) {
g_log->warn("[virtio_net] dma API unavailable");
} else {
g_io_base = (u16)(bar0 & 0xfffcU);
log_hex("[virtio_net] legacy io_base=", g_io_base);
if (vdev_negotiate() != 0) {
g_log->warn("[virtio_net] feature negotiation failed");
} else if (vq_setup(api->dma, &g_rxq, 0) != 0) {
g_log->warn("[virtio_net] RXQ setup failed");
} else if (vq_setup(api->dma, &g_txq, 1) != 0) {
g_log->warn("[virtio_net] TXQ setup failed");
} else if (rx_prefill(api->dma) != 0) {
g_log->warn("[virtio_net] RX prefill failed");
} else if (tx_buffers_alloc(api->dma) != 0) {
g_log->warn("[virtio_net] TX buffer alloc failed");
} else {
log_hex("[virtio_net] rxq size=", g_rxq.size);
log_hex("[virtio_net] rxq phys=", g_rxq.phys);
log_hex("[virtio_net] txq size=", g_txq.size);
log_hex("[virtio_net] txq phys=", g_txq.phys);
/* MAC: device config @ io_base+0x14（协商 MAC feature 后有效） */
for (u32 i = 0; i < 6; i++) {
g_mac[i] = dkm_inb((u16)(g_io_base + VP_NET_CFG_MAC + i));
}
u8 mac_sum = 0;
for (u32 i = 0; i < 6; i++) mac_sum |= g_mac[i];
if (mac_sum == 0 || (g_mac[0] & 1u)) {
/* 全 0 或组播位异常 → fallback */
g_mac[0] = 0x52; g_mac[1] = 0x54; g_mac[2] = 0x00;
g_mac[3] = 0x12; g_mac[4] = 0x34; g_mac[5] = 0x57;
g_log->warn("[virtio_net] invalid device MAC, using fallback");
}
log_mac("[virtio_net] MAC=", g_mac);
/* DRIVER_OK：设备开始处理队列；随后 kick RXQ 让设备吃掉预填 buffer */
vdev_set_status(VS_ACKNOWLEDGE | VS_DRIVER | VS_FEATURES_OK | VS_DRIVER_OK);
vq_notify(&g_rxq);
g_ready = 1;
datapath_ok = 1;
g_log->info("[virtio_net] DRIVER_OK set; virtqueues live");
}
}
/* ---- IRQ 注册（辅助统计；数据路径为纯轮询，不依赖中断） ----
* 注意: kernel irq_register 按 IDT vector 索引（PIC IRQn → vector 0x20+n），
* 传裸 IRQ 号会落到异常向量槽位永不触发（e1000 当前如此）。
* PIC 未解屏蔽该线时自动退化为纯轮询。 */
if (datapath_ok && api->irq_register && irq_line > 0 && irq_line < 16) {
api->irq_register((u8)(irq_line + 0x20u), (void *)(uintptr_t)virtio_net_irq);
g_log->info("[virtio_net] IRQ handler registered (vector = irq + 0x20)");
} else {
g_log->info("[virtio_net] IRQ not registered (polling mode)");
}
/* ---- 注册 netdev ---- */
if (api->net && api->net->register_device) {
struct dkm_net_device_desc netdev;
zero_bytes(&netdev, sizeof(netdev));
netdev.name = "virtio-net";
for (int i = 0; i < 6; i++) netdev.mac[i] = 0;
netdev.flags = DKM_NET_F_LINK_UP;
for (u32 i = 0; i < 6; i++) netdev.mac[i] = g_mac[i];
netdev.ctx = 0;
netdev.tx = virtio_net_tx; /* stub: 安全占位, 不 NULL */
netdev.rx_poll = virtio_net_rx_poll;
if (datapath_ok) {
netdev.flags = DKM_NET_F_LINK_UP | DKM_NET_F_TX_READY | DKM_NET_F_RX_READY;
netdev.tx = virtio_net_tx;
netdev.rx_poll = virtio_net_rx_poll;
} else {
/* 安全占位：flags 仅 LINK_UP，netman 不会选它跑 DHCP（与历史行为一致） */
netdev.flags = DKM_NET_F_LINK_UP;
netdev.tx = virtio_net_tx_stub;
netdev.rx_poll = virtio_net_rx_poll_stub;
}
int net_index = api->net->register_device(&netdev);
log_hex("[virtio_net] netdev register rc=", (u64)(i64)net_index);
log_hex("[virtio_net] netdev registered, index=", (u64)(i64)net_index);
} else {
g_log->warn("[virtio_net] net API unavailable; netdev not registered");
}
g_log->info("[virtio_net] discovery only; virtqueues disabled");
/* ---- boot 自测（证明 TX/RX 全链路，失败仅记日志） ---- */
virtio_net_selftest();
log_hex("[virtio_net] stat tx_ok=", g_stat_tx_ok);
log_hex("[virtio_net] stat rx_ok=", g_stat_rx_ok);
g_log->info("[virtio_net] driver ready");
return 0;
}
@@ -278,5 +939,11 @@ int driver_init(const struct dkm_kernel_api *api,
__attribute__((visibility("default")))
int driver_exit(struct dkm_driver_handle *handle) {
(void)handle;
/* reset 设备：停止一切 DMA 与队列活动 */
if (g_io_base) {
vdev_set_status(0);
}
g_ready = 0;
g_io_base = 0;
return 0;
}
CODE/DKM/xhci/xhci.c
+1783
View File
@@ -0,0 +1,1783 @@
/* DKM xHCI Driver — USB xHCI host controller + Mass Storage Class (BOT/SCSI)
*
* Stage 1 optional storage driver, depends on "pci" and "irq", provides "block".
*
* Architecture:
* 1. PCI discovery: class=0x0C/03/30 (Serial Bus / USB / xHCI)
* 2. MMIO mapping: 64-bit BAR0 via mm_map_mmio or HHDM fallback
* 3. Controller init: BIOS handoff, HC reset, parse capabilities
* 4. Ring setup: Command Ring, Event Ring, ERST, DCBAA, Scratchpad
* 5. Port scan: detect connected devices, port reset
* 6. USB enumeration: Enable Slot, Address Device, GET_DESCRIPTOR
* 7. MSC protocol: BOT (Bulk-Only Transport) + SCSI commands
* 8. Block provider: register usb0 via kernel_api.block
*
* Synchronous polling model (no IRQ/MSI in first version).
* All hardware waits use TSC deadline — no infinite loops.
*/
#include "../dkm_shared.h"
#include "../dkm_instr.h"
DKM_STAT_DECL(xhci_pci_found);
DKM_STAT_DECL(xhci_port_reset_ok);
DKM_STAT_DECL(xhci_msc_found);
DKM_STAT_DECL(xhci_block_reads);
DKM_STAT_DECL(xhci_block_writes);
static const char *const g_depends[] = { "pci", "irq" };
static const char *const g_provides[] = { "block" };
__attribute__((visibility("default"), used))
const struct dkm_driver_desc driver_desc = {
.magic = DKM_DRIVER_MAGIC,
.abi_version = DKM_ABI_VERSION,
.desc_size = sizeof(struct dkm_driver_desc),
.name = "xhci",
.version = "0.1.0",
.vendor = "Deshab",
.driver_class = 6, /* DKM_CLASS_STORAGE */
.stage = 1,
.flags = 0,
.priority = 0,
.depends = g_depends,
.depends_count = 2,
.provides = g_provides,
.provides_count = 1,
.min_kernel_abi = 1,
.feature_bits = 0,
.reserved0 = 0,
.reserved1 = 0
};
/* ================================================================
* Constants — PCI, xHCI registers, TRB types, USB
* ================================================================ */
#define XHCI_CLASS_SERIAL 0x0C
#define XHCI_SUBCLASS_USB 0x03
#define XHCI_PROGIF_XHCI 0x30
/* Capability Registers (MMIO + 0x00) */
#define XHCI_CAP_HCSPARAMS1 0x00
#define XHCI_CAP_HCSPARAMS2 0x04
#define XHCI_CAP_HCSPARAMS3 0x08
#define XHCI_CAP_HCCPARAMS1 0x0C
#define XHCI_CAP_DBOFF 0x14
#define XHCI_CAP_RTSOFF 0x18
/* Operational Registers (MMIO + cap_length) */
#define XHCI_OP_USBCMD 0x00
#define XHCI_OP_USBSTS 0x04
#define XHCI_OP_PAGESIZE 0x08
#define XHCI_OP_DNCTRL 0x14
#define XHCI_OP_CRCR_LO 0x18
#define XHCI_OP_CRCR_HI 0x1C
#define XHCI_OP_DCBAAP_LO 0x30
#define XHCI_OP_DCBAAP_HI 0x34
#define XHCI_OP_CONFIG 0x38
/* Port Registers (MMIO + 0x400 + port * 0x10) */
#define XHCI_PORT_BASE_OFF 0x400
#define XHCI_PORT_STRIDE 0x10
#define XHCI_PORT_SC 0x00
/* Runtime Registers (MMIO + rtsoff) */
#define XHCI_RT_MFINDEX 0x00
#define XHCI_RT_IR0_IMAN 0x20
#define XHCI_RT_IR0_IMOD 0x24
#define XHCI_RT_IR0_ERSTSZ 0x28
#define XHCI_RT_IR0_ERSTBA_LO 0x30
#define XHCI_RT_IR0_ERSTBA_HI 0x34
#define XHCI_RT_IR0_ERDP_LO 0x38
#define XHCI_RT_IR0_ERDP_HI 0x3C
/* USBCMD bits */
#define USBCMD_RS (1u << 0)
#define USBCMD_RST (1u << 1)
#define USBCMD_INTE (1u << 2)
/* USBSTS bits */
#define USBSTS_HCH (1u << 0)
#define USBSTS_EINT (1u << 3)
#define USBSTS_CNR (1u << 11)
/* PORTSC bits */
#define PORTSC_CCS (1u << 0)
#define PORTSC_PED (1u << 1)
#define PORTSC_OCA (1u << 3)
#define PORTSC_PR (1u << 4)
#define PORTSC_PLS_SHIFT 5
#define PORTSC_PLS_MASK 0xF
#define PORTSC_SPEED_SHIFT 17
#define PORTSC_SPEED_MASK 0x7
/* TRB types */
#define TRB_TYPE_NORMAL 0x01
#define TRB_TYPE_SETUP_STAGE 0x02
#define TRB_TYPE_DATA_STAGE 0x03
#define TRB_TYPE_STATUS_STAGE 0x04
#define TRB_TYPE_LINK 0x06
#define TRB_TYPE_ENABLE_SLOT 0x09
#define TRB_TYPE_DISABLE_SLOT 0x0A
#define TRB_TYPE_ADDRESS_DEV 0x0B
#define TRB_TYPE_CONFIGURE_EP 0x0C
#define TRB_TYPE_NOOP 0x17
/* Event TRB types — same namespace as command/transfer TRB types.
* Per xHCI spec §6.4: these are the TRB Type field values in the
* Event TRB control dword [15:10]. */
#define EVT_TYPE_TRANSFER 0x01 /* Transfer Event */
#define EVT_TYPE_CMD_COMPLETE 0x02 /* Command Completion Event */
#define EVT_TYPE_PORT_CHANGE 0x21 /* Port Status Change Event */
/* TRB control field helpers */
#define TRB_TYPE_SHIFT 10
#define TRB_TYPE_MASK 0x3F
#define TRB_IOC (1u << 5) /* Interrupt On Completion */
#define TRB_CH (1u << 6) /* Chain bit */
#define TRB_ENT (1u << 7) /* Evaluate Next TRB (for Link) */
#define TRB_ISP (1u << 8) /* Interrupt on Short Packet */
#define TRB_CYCLE (1u << 0)
#define TRB_TC (1u << 1) /* Toggle Cycle (for Link) */
/* Setup Stage TRB direction */
#define SETUP_DIR_OUT 0
#define SETUP_DIR_IN 1
/* USB speeds */
#define USB_SPEED_FULL 1
#define USB_SPEED_LOW 2
#define USB_SPEED_HIGH 3
#define USB_SPEED_SUPER 4
/* Endpoint types for EP Context */
#define EP_TYPE_CTRL_OUT 0
#define EP_TYPE_CTRL_IN 4
#define EP_TYPE_BULK_OUT 2
#define EP_TYPE_BULK_IN 6
/* Slot states */
#define SLOT_STATE_DISABLED 0
#define SLOT_STATE_ENABLED 1
#define SLOT_STATE_ADDRESSED 2
#define SLOT_STATE_CONFIGURED 3
/* Ring sizes */
#define CMD_RING_SIZE 256
#define EVT_RING_SIZE 256
#define XFER_RING_SIZE 256
/* xHCI MMIO mapping size (64 KB covers most controllers) */
#define XHCI_MAP_SIZE 0x10000
/* BOT (Bulk-Only Transport) signatures */
#define BOT_CBW_SIG 0x43425355u /* "USBC" */
#define BOT_CSW_SIG 0x53425355u /* "USBS" */
/* SCSI opcodes */
#define SCSI_TEST_UNIT_READY 0x00
#define SCSI_INQUIRY 0x12
#define SCSI_READ_CAPACITY10 0x25
#define SCSI_READ10 0x28
#define SCSI_WRITE10 0x2A
/* ================================================================
* Data structures
* ================================================================ */
typedef struct {
u64 parameter; /* DW0-DW1 */
u32 status; /* DW2 */
u32 control; /* DW3 */
} xhci_trb;
/* TRB Ring */
typedef struct {
struct dkm_dma_buffer dma;
u32 size; /* TRB count */
u32 enqueue; /* producer index */
u32 ccs; /* producer Cycle State */
} xhci_ring;
/* ERST entry */
typedef struct {
u64 seg_addr;
u32 seg_size;
u32 rsvd;
} xhci_erst_entry;
/* MSC device state */
typedef struct {
u8 slot_id;
u8 speed;
u8 bulk_out_ep; /* EP number for Bulk OUT */
u8 bulk_in_ep; /* EP number for Bulk IN */
u16 bulk_out_mps; /* Max Packet Size */
u16 bulk_in_mps; /* Max Packet Size */
u32 cbw_tag; /* BOT tag counter */
u64 sector_count;
u32 sector_size;
int ready;
} xhci_msc_dev;
/* ================================================================
* Global state
* ================================================================ */
static const struct dkm_kernel_api *g_api;
static const struct dkm_log_api *g_log;
static volatile u8 *g_mmio; /* MMIO base virtual address */
static u32 g_cap_length; /* Capability register space length */
static u32 g_doorbell_off; /* Doorbell register offset */
static u32 g_runtime_off; /* Runtime register space offset */
static u32 g_max_slots;
static u32 g_max_ports;
static u32 g_max_intrs;
static u32 g_ctx_size; /* 32 or 64 bytes */
static u32 g_page_size; /* controller page size in bytes */
static u32 g_spb_count; /* scratchpad buffer count */
static u32 g_port_base; /* absolute MMIO offset of port register set (cap_length + 0x400) */
static u32 g_ppc; /* Per-Port Power Control (HCCPARAMS1 bit 3) */
static xhci_ring g_cmd_ring;
static xhci_ring g_evt_ring;
static u32 g_evt_ccs; /* consumer cycle state for event ring */
static struct dkm_dma_buffer g_erst; /* ERST DMA buffer */
static struct dkm_dma_buffer g_dcbaa; /* DCBAA DMA buffer */
static struct dkm_dma_buffer g_sp_bufs; /* Scratchpad buffer array */
/* Scratchpad individual pages — max 32 scratchpads for safety */
static struct dkm_dma_buffer g_sp_pages[32];
/* Per-device DMA buffers */
static struct dkm_dma_buffer g_input_ctx; /* Input Context */
static struct dkm_dma_buffer g_dev_ctx; /* Device Context (output) */
static xhci_ring g_ep0_ring; /* EP0 control transfer ring */
static xhci_ring g_bulk_out_ring;
static xhci_ring g_bulk_in_ring;
static struct dkm_dma_buffer g_xfer_buf; /* Data transfer bounce buffer */
static struct dkm_dma_buffer g_csw_buf; /* CSW receive buffer */
static xhci_msc_dev g_msc;
/* PCI device location */
static u8 g_pci_bus, g_pci_dev, g_pci_func;
/* ================================================================
* Utility functions
* ================================================================ */
static void log_hex(const char *prefix, u64 value) {
static const char hex[] = "0123456789abcdef";
char buf[19];
u32 pos = 0;
buf[pos++] = '0';
buf[pos++] = 'x';
for (int i = 15; i >= 0; i--)
buf[pos++] = hex[(value >> (i * 4)) & 0xf];
buf[pos] = 0;
g_log->info(prefix);
g_log->info(buf);
}
static void xhci_zero(void *ptr, u32 len) {
u8 *p = (u8 *)ptr;
for (u32 i = 0; i < len; i++) p[i] = 0;
}
static void xhci_memcpy(void *dst, const void *src, u32 len) {
u8 *d = (u8 *)dst;
const u8 *s = (const u8 *)src;
for (u32 i = 0; i < len; i++) d[i] = s[i];
}
/* MMIO read/write */
static u32 mmio_read32(u32 off) {
return *(volatile u32 *)(void *)(g_mmio + off);
}
static void mmio_write32(u32 off, u32 value) {
*(volatile u32 *)(void *)(g_mmio + off) = value;
}
static u64 mmio_read64(u32 off) {
u32 lo = mmio_read32(off);
u32 hi = mmio_read32(off + 4);
return ((u64)hi << 32) | lo;
}
static void mmio_write64(u32 off, u64 value) {
mmio_write32(off, (u32)(value & 0xffffffffu));
mmio_write32(off + 4, (u32)(value >> 32));
}
/* Memory barrier (compiler only — x86 TSO guarantees HW ordering) */
static __inline__ void xhci_mb(void) {
__asm__ volatile("" ::: "memory");
}
/* ================================================================
* Phase 1: PCI discovery + MMIO mapping
* ================================================================ */
static int xhci_find(u8 *out_bus, u8 *out_dev, u8 *out_func) {
for (u16 bus = 0; bus < 256; bus++) {
for (u8 dev = 0; dev < 32; dev++) {
u32 vd = dkm_pci_read((u8)bus, dev, 0, PCI_VENDOR_ID);
if ((vd & 0xffff) == 0xffff) continue;
u8 header = (u8)(dkm_pci_read((u8)bus, dev, 0, PCI_HEADER) >> 16);
u8 func_count = (header & 0x80) ? 8 : 1;
for (u8 func = 0; func < func_count; func++) {
u32 vd2 = dkm_pci_read((u8)bus, dev, func, PCI_VENDOR_ID);
if ((vd2 & 0xffff) == 0xffff) continue;
u32 class_reg = dkm_pci_read((u8)bus, dev, func, 0x08);
u8 prog_if = (u8)((class_reg >> 8) & 0xff);
u8 subclass = (u8)((class_reg >> 16) & 0xff);
u8 class_code = (u8)((class_reg >> 24) & 0xff);
if (class_code == XHCI_CLASS_SERIAL &&
subclass == XHCI_SUBCLASS_USB &&
prog_if == XHCI_PROGIF_XHCI) {
*out_bus = (u8)bus;
*out_dev = dev;
*out_func = func;
return 0;
}
}
}
}
return -1;
}
static volatile u8 *xhci_map_bar(const struct dkm_kernel_api *api, u64 bar_phys) {
u64 bar_virt = api->hhdm_offset + bar_phys;
/* Prefer mm_map_mmio independent window */
if (api->mm_map_mmio) {
void *mapped = api->mm_map_mmio(bar_phys, XHCI_MAP_SIZE);
if (mapped) {
g_log->info("[xhci] BAR mapped via mm_map_mmio");
return (volatile u8 *)mapped;
}
g_log->warn("[xhci] mm_map_mmio failed; trying HHDM fallback");
}
/* Check if HHDM covers this BAR */
if (api->mmio && api->mmio->is_mapped) {
int level = api->mmio->is_mapped(bar_virt);
if (level > 0) {
g_log->info("[xhci] HHDM covers BAR; direct access");
return (volatile u8 *)(uintptr_t)bar_virt;
}
/* Map via page tables */
if (api->mmio->map_mmio) {
int rc = api->mmio->map_mmio(bar_phys, XHCI_MAP_SIZE);
if (rc == 0 && api->mmio->is_mapped(bar_virt) > 0) {
g_log->info("[xhci] BAR mapped via page tables");
return (volatile u8 *)(uintptr_t)bar_virt;
}
}
}
/* Low BAR fallback */
if (bar_phys < 0x100000000ULL) {
g_log->warn("[xhci] assuming HHDM covers low BAR");
return (volatile u8 *)(uintptr_t)bar_virt;
}
g_log->error("[xhci] BAR MMIO unavailable");
return (volatile u8 *)0;
}
/* ================================================================
* Phase 1: BIOS Handoff via Extended Capabilities
* ================================================================ */
static void xhci_bios_handoff(void) {
u32 hcc1 = mmio_read32(XHCI_CAP_HCCPARAMS1);
u32 ext_ptr = (hcc1 >> 16) & 0xFFFF;
if (!ext_ptr) return;
/* Walk extended capabilities (offsets in 32-bit words from MMIO base) */
u32 off = ext_ptr;
for (int i = 0; i < 16 && off != 0; i++) {
u32 dw0 = mmio_read32(off * 4);
u8 id = (u8)(dw0 & 0xFF);
u8 next = (u8)((dw0 >> 8) & 0xFF);
if (id == 1) {
/* USB Legacy Support Capability */
u32 legsup = mmio_read32(off * 4 + 4);
log_hex("[xhci] USBLEGSUP=", legsup);
if (legsup & 0x0000FFFFu) {
/* BIOS owns controller — request ownership */
mmio_write32(off * 4 + 4, legsup | (1u << 16));
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 1000;
while (dkm_rdtsc() < deadline) {
u32 cur = mmio_read32(off * 4 + 4);
if (!(cur & 0x0000FFFFu)) {
g_log->info("[xhci] BIOS handoff complete");
/* Clear SMI triggers */
mmio_write32(off * 4 + 8, 0);
break;
}
__asm__ volatile("pause");
}
}
}
off = next ? (off + next) : 0;
}
}
/* ================================================================
* Phase 1: Controller reset and init
* ================================================================ */
static int xhci_controller_reset(void) {
u32 usbcmd = mmio_read32(g_cap_length + XHCI_OP_USBCMD);
u32 usbsts = mmio_read32(g_cap_length + XHCI_OP_USBSTS);
/* If controller is running, halt it first */
if (!(usbsts & USBSTS_HCH)) {
mmio_write32(g_cap_length + XHCI_OP_USBCMD, usbcmd & ~USBCMD_RS);
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 1000;
while (dkm_rdtsc() < deadline) {
if (mmio_read32(g_cap_length + XHCI_OP_USBSTS) & USBSTS_HCH) break;
__asm__ volatile("pause");
}
}
/* Wait for CNR to clear */
{
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 1000;
while (dkm_rdtsc() < deadline) {
if (!(mmio_read32(g_cap_length + XHCI_OP_USBSTS) & USBSTS_CNR)) break;
__asm__ volatile("pause");
}
}
/* Assert HC reset */
usbcmd = mmio_read32(g_cap_length + XHCI_OP_USBCMD);
mmio_write32(g_cap_length + XHCI_OP_USBCMD, usbcmd | USBCMD_RST);
/* Wait for reset to complete (RST bit clears) */
{
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 2000;
while (dkm_rdtsc() < deadline) {
usbcmd = mmio_read32(g_cap_length + XHCI_OP_USBCMD);
if (!(usbcmd & USBCMD_RST)) {
g_log->info("[xhci] controller reset complete");
return 0;
}
__asm__ volatile("pause");
}
}
g_log->error("[xhci] controller reset timeout");
return -1;
}
static int xhci_parse_caps(void) {
/* Read CAPLENGTH (low byte of MMIO+0) */
g_cap_length = mmio_read32(0) & 0xFF;
u32 hcs1 = mmio_read32(XHCI_CAP_HCSPARAMS1);
u32 hcs2 = mmio_read32(XHCI_CAP_HCSPARAMS2);
u32 hcc1 = mmio_read32(XHCI_CAP_HCCPARAMS1);
g_max_slots = hcs1 & 0xFF;
g_max_intrs = (hcs1 >> 8) & 0x7FF;
g_max_ports = (hcs1 >> 24) & 0xFF;
/* Context size: CSZ bit4 of HCCPARAMS1 */
g_ctx_size = (hcc1 & (1u << 4)) ? 64 : 32;
/* Per-Port Power Control: PPC bit3 of HCCPARAMS1 */
g_ppc = (hcc1 >> 3) & 1;
/* Scratchpad buffer count from HCSPARAMS2 */
g_spb_count = ((hcs2 >> 21) & 0x1F) | ((hcs2 >> 27) & 0x1F);
/*
* Port Register Set offset: absolute from MMIO base.
* xHCI spec: "Port Register Sets are located at offset 0x400
* from the beginning of the Operational Register Space."
* Operational Register Space starts at offset CAPLENGTH.
* Therefore absolute port offset = CAPLENGTH + 0x400.
* Verified against QEMU source (XHCI_REGS_OFFSET_PORT =
* XHCI_REGS_OFFSET_OPER + XHCI_REGS_LENGTH_OPER) and Linux
* kernel (op_regs + PORTSC_OFFSET where op_regs = base + cap_length).
*/
g_port_base = g_cap_length + XHCI_PORT_BASE_OFF;
/* Doorbell and Runtime offsets — full 32-bit, clear reserved low bits */
g_doorbell_off = mmio_read32(XHCI_CAP_DBOFF) & 0xFFFFFFF0u;
g_runtime_off = mmio_read32(XHCI_CAP_RTSOFF) & 0xFFFFFFE0u;
/* Page size */
u32 ps_reg = mmio_read32(g_cap_length + XHCI_OP_PAGESIZE);
g_page_size = 1u << ((ps_reg & 0xF) + 12);
log_hex("[xhci] CAPLEN=", g_cap_length);
log_hex("[xhci] MaxSlots=", g_max_slots);
log_hex("[xhci] MaxPorts=", g_max_ports);
log_hex("[xhci] MaxIntrs=", g_max_intrs);
log_hex("[xhci] ContextSize=", g_ctx_size);
log_hex("[xhci] PPC=", g_ppc);
log_hex("[xhci] PortBaseOff=", g_port_base);
log_hex("[xhci] ScratchpadBufs=", g_spb_count);
log_hex("[xhci] DBOFF=", g_doorbell_off);
log_hex("[xhci] RTSOFF=", g_runtime_off);
log_hex("[xhci] PageSize=", g_page_size);
if (g_max_slots == 0 || g_max_ports == 0) {
g_log->error("[xhci] invalid capability parameters");
return -1;
}
if (g_max_intrs == 0) {
g_log->warn("[xhci] MaxIntrs=0 (no interrupt register sets); polling mode only");
}
return 0;
}
/* ================================================================
* Phase 2: TRB Ring management
* ================================================================ */
static int xhci_ring_init(xhci_ring *ring, u32 size) {
if (!g_api->dma) return -1;
/* Allocate one page (must be 4K aligned for DMA) */
if (g_api->dma->alloc_pages(1, 4096, 0x100000000ULL, &ring->dma) != 0)
return -2;
xhci_zero(ring->dma.virt, ring->dma.size);
ring->size = size;
ring->enqueue = 0;
ring->ccs = 1; /* Producer Cycle State starts at 1 */
/* Append Link TRB at end of ring */
xhci_trb *trbs = (xhci_trb *)ring->dma.virt;
u32 link_idx = size - 1;
trbs[link_idx].parameter = ring->dma.phys;
trbs[link_idx].status = 0;
trbs[link_idx].control = (TRB_TYPE_LINK << TRB_TYPE_SHIFT) | TRB_TC | TRB_CYCLE;
return 0;
}
/* Enqueue a TRB onto a ring and return the enqueue index before advance */
static u32 xhci_ring_enqueue(xhci_ring *ring, u64 param, u32 status, u32 control) {
xhci_trb *trbs = (xhci_trb *)ring->dma.virt;
u32 idx = ring->enqueue;
/* Set cycle bit in control */
control &= ~TRB_CYCLE;
if (ring->ccs) control |= TRB_CYCLE;
trbs[idx].parameter = param;
trbs[idx].status = status;
trbs[idx].control = control;
/* Advance enqueue */
ring->enqueue = (idx + 1) % (ring->size - 1); /* -1 to skip Link TRB */
/* If we wrapped, toggle CCS */
if (idx + 1 >= ring->size - 1)
ring->ccs ^= 1;
return idx;
}
/* Ring a doorbell: slot_id=0 for command, otherwise endpoint doorbell */
static void xhci_ring_doorbell(u32 slot_id, u32 target) {
mmio_write32(g_doorbell_off + slot_id * 0x20, target);
}
/* ================================================================
* Phase 2: Event Ring polling
* ================================================================ */
/* Poll event ring for a specific TRB type with TSC timeout.
* Returns 0 on success, fills out_param/out_status/out_control.
* Returns -1 on timeout, -2 on error completion code. */
static int xhci_poll_event(u32 expected_type, u64 timeout_ms,
u64 *out_param, u32 *out_status, u32 *out_control) {
xhci_trb *trbs = (xhci_trb *)g_evt_ring.dma.virt;
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * timeout_ms;
for (;;) {
xhci_trb *evt = &trbs[g_evt_ring.enqueue];
u32 ctrl = evt->control;
u8 cycle = (u8)(ctrl & TRB_CYCLE);
/* Check if this TRB is owned by us (consumer) */
if (cycle == (u8)g_evt_ccs) {
u8 type = (u8)((ctrl >> TRB_TYPE_SHIFT) & TRB_TYPE_MASK);
/* Process this event */
if (type == expected_type) {
if (out_param) *out_param = evt->parameter;
if (out_status) *out_status = evt->status;
if (out_control)*out_control= evt->control;
/* Advance dequeue */
g_evt_ring.enqueue = (g_evt_ring.enqueue + 1) % (g_evt_ring.size - 1);
if (g_evt_ring.enqueue == 0) g_evt_ccs ^= 1;
/* Update ERDP */
u64 erdp = g_evt_ring.dma.phys + (u64)g_evt_ring.enqueue * sizeof(xhci_trb);
mmio_write64(g_runtime_off + XHCI_RT_IR0_ERDP_LO, erdp | g_evt_ccs);
/* Check completion code */
u32 cc = (evt->status >> 24) & 0xFF;
if (cc != 1) { /* 1 = Success */
if (cc != 0) {
log_hex("[xhci] event completion code=", cc);
return -2;
}
}
return 0;
}
/* Unexpected event type — handle or skip */
if (type == EVT_TYPE_PORT_CHANGE) {
/* Port Status Change Event: clear PORTSC change bits */
u32 port_id = (u32)((evt->parameter >> 24) & 0xFF);
if (port_id > 0 && port_id <= g_max_ports) {
u32 port_off = g_port_base + (port_id - 1) * XHCI_PORT_STRIDE + XHCI_PORT_SC;
/* Read PORTSC and write back to clear change bits (W1C) */
u32 portsc = mmio_read32(port_off);
mmio_write32(port_off, portsc);
log_hex("[xhci] port change event, port=", port_id);
}
} else {
log_hex("[xhci] unexpected event type=", type);
}
g_evt_ring.enqueue = (g_evt_ring.enqueue + 1) % (g_evt_ring.size - 1);
if (g_evt_ring.enqueue == 0) g_evt_ccs ^= 1;
u64 erdp2 = g_evt_ring.dma.phys + (u64)g_evt_ring.enqueue * sizeof(xhci_trb);
mmio_write64(g_runtime_off + XHCI_RT_IR0_ERDP_LO, erdp2 | g_evt_ccs);
continue;
}
if (dkm_rdtsc() > deadline) {
g_log->warn("[xhci] event poll timeout");
return -1;
}
__asm__ volatile("pause");
}
}
/* ================================================================
* Phase 2: Command submission
* ================================================================ */
static int xhci_cmd_submit(u64 param, u32 status, u32 control,
u64 timeout_ms,
u64 *out_param, u32 *out_status) {
/* Clear Event Ring interrupt pending */
mmio_write32(g_runtime_off + XHCI_RT_IR0_IMAN,
mmio_read32(g_runtime_off + XHCI_RT_IR0_IMAN) | 2u);
/* Enqueue command TRB */
xhci_ring_enqueue(&g_cmd_ring, param, status, control);
xhci_mb();
/* Debug: verify CRCR and command ring state */
log_hex("[xhci] CRCR=", mmio_read64(g_cap_length + XHCI_OP_CRCR_LO));
log_hex("[xhci] cmd ring enqueue=", g_cmd_ring.enqueue);
log_hex("[xhci] cmd ring ccs=", g_cmd_ring.ccs);
xhci_ring_doorbell(0, 0); /* Command ring doorbell */
/* Debug: check USBSTS after doorbell */
log_hex("[xhci] USBSTS after db=", mmio_read32(g_cap_length + XHCI_OP_USBSTS));
/* Poll for Command Completion Event */
/* Debug: dump event ring head entry */
{
xhci_trb *evts = (xhci_trb *)g_evt_ring.dma.virt;
log_hex("[xhci] evt ring head=", g_evt_ring.enqueue);
log_hex("[xhci] evt[0] ctrl=", evts[g_evt_ring.enqueue].control);
}
int rc = xhci_poll_event(EVT_TYPE_CMD_COMPLETE, timeout_ms,
out_param, out_status, 0);
/* Clear EINT */
mmio_write32(g_cap_length + XHCI_OP_USBSTS,
mmio_read32(g_cap_length + XHCI_OP_USBSTS) | USBSTS_EINT);
return rc;
}
/* ================================================================
* Phase 2: Setup controller rings + DCBAA + Scratchpad
* ================================================================ */
static int xhci_setup_rings(void) {
if (!g_api->dma) {
g_log->warn("[xhci] DMA API unavailable");
return -1;
}
/* Command Ring */
if (xhci_ring_init(&g_cmd_ring, CMD_RING_SIZE) != 0) {
g_log->error("[xhci] cmd ring alloc failed");
return -2;
}
log_hex("[xhci] cmd ring phys=", g_cmd_ring.dma.phys);
/* Event Ring */
if (xhci_ring_init(&g_evt_ring, EVT_RING_SIZE) != 0) {
g_log->error("[xhci] evt ring alloc failed");
return -3;
}
g_evt_ccs = 1; /* Consumer Cycle State starts at 1 */
log_hex("[xhci] evt ring phys=", g_evt_ring.dma.phys);
/* ERST (Event Ring Segment Table) */
if (g_api->dma->alloc_pages(1, 4096, 0x100000000ULL, &g_erst) != 0) {
g_log->error("[xhci] ERST alloc failed");
return -4;
}
xhci_erst_entry *erst = (xhci_erst_entry *)g_erst.virt;
xhci_zero(erst, sizeof(xhci_erst_entry));
erst->seg_addr = g_evt_ring.dma.phys;
erst->seg_size = EVT_RING_SIZE;
/* DCBAA (Device Context Base Address Array) */
u32 dcbaa_size = (g_max_slots + 1) * 8;
u32 dcbaa_pages = (dcbaa_size + 4095) / 4096;
if (dcbaa_pages == 0) dcbaa_pages = 1;
if (g_api->dma->alloc_pages(dcbaa_pages, 4096, 0x100000000ULL, &g_dcbaa) != 0) {
g_log->error("[xhci] DCBAA alloc failed");
return -5;
}
xhci_zero(g_dcbaa.virt, g_dcbaa.size);
log_hex("[xhci] DCBAA phys=", g_dcbaa.phys);
/* Scratchpad Buffer Array (if needed) */
if (g_spb_count > 0) {
u32 sp_arr_size = g_spb_count * 8;
u32 sp_arr_pages = (sp_arr_size + 4095) / 4096;
if (sp_arr_pages == 0) sp_arr_pages = 1;
if (g_api->dma->alloc_pages(sp_arr_pages, 4096, 0x100000000ULL, &g_sp_bufs) != 0) {
g_log->error("[xhci] scratchpad array alloc failed");
return -6;
}
xhci_zero(g_sp_bufs.virt, g_sp_bufs.size);
/* Allocate individual scratchpad buffer pages */
u32 sp_count = g_spb_count;
if (sp_count > 32) sp_count = 32; /* safety cap */
for (u32 i = 0; i < sp_count; i++) {
if (g_api->dma->alloc_pages(1, 4096, 0x100000000ULL, &g_sp_pages[i]) != 0) {
g_log->error("[xhci] scratchpad page alloc failed");
return -7;
}
xhci_zero(g_sp_pages[i].virt, g_sp_pages[i].size);
/* Fill scratchpad buffer array with physical addresses */
u64 *sp_arr = (u64 *)g_sp_bufs.virt;
sp_arr[i] = g_sp_pages[i].phys;
}
/* DCBAA slot 0 points to scratchpad buffer array */
u64 *dcbaa_entries = (u64 *)g_dcbaa.virt;
dcbaa_entries[0] = g_sp_bufs.phys;
log_hex("[xhci] SPB array phys=", g_sp_bufs.phys);
}
return 0;
}
static int xhci_controller_start(void) {
u32 cap_off = g_cap_length;
/* Configure Command Ring */
u64 crcr = g_cmd_ring.dma.phys | (u64)g_cmd_ring.ccs;
mmio_write64(cap_off + XHCI_OP_CRCR_LO, crcr);
/* Configure DCBAAP */
mmio_write64(cap_off + XHCI_OP_DCBAAP_LO, g_dcbaa.phys);
/* Configure MaxSlotsEn */
mmio_write32(cap_off + XHCI_OP_CONFIG, g_max_slots);
/* Configure Event Ring */
mmio_write32(g_runtime_off + XHCI_RT_IR0_ERSTSZ, 1);
mmio_write64(g_runtime_off + XHCI_RT_IR0_ERSTBA_LO, g_erst.phys);
u64 erdp = g_evt_ring.dma.phys | (u64)g_evt_ccs;
mmio_write64(g_runtime_off + XHCI_RT_IR0_ERDP_LO, erdp);
/* Enable interrupt (IMAN.IE=1) */
mmio_write32(g_runtime_off + XHCI_RT_IR0_IMAN,
mmio_read32(g_runtime_off + XHCI_RT_IR0_IMAN) | (1u << 1));
/* Enable controller interrupts + start controller */
u32 usbcmd = mmio_read32(cap_off + XHCI_OP_USBCMD);
usbcmd |= USBCMD_INTE | USBCMD_RS;
mmio_write32(cap_off + XHCI_OP_USBCMD, usbcmd);
/* Wait for HCH to clear (controller running) */
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 2000;
while (dkm_rdtsc() < deadline) {
u32 usbsts = mmio_read32(cap_off + XHCI_OP_USBSTS);
if (!(usbsts & USBSTS_HCH)) {
g_log->info("[xhci] controller started");
return 0;
}
__asm__ volatile("pause");
}
g_log->error("[xhci] controller start timeout");
return -1;
}
/* ================================================================
* Phase 3: Port detection and reset
* ================================================================ */
static int xhci_reset_port(u32 port_idx) {
u32 port_off = g_port_base + port_idx * XHCI_PORT_STRIDE + XHCI_PORT_SC;
u32 portsc = mmio_read32(port_off);
/* Check if device is connected */
if (!(portsc & PORTSC_CCS)) return -1;
/* Already enabled? */
if (portsc & PORTSC_PED) return 0;
/* Clear any change bits (W1C), then assert port reset */
mmio_write32(port_off, portsc | PORTSC_PR);
/* Wait for PED (Port Enabled) with 500ms timeout */
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 500;
while (dkm_rdtsc() < deadline) {
portsc = mmio_read32(port_off);
if (portsc & PORTSC_PED) {
/* Clear port change bits (W1C: write 1 to clear) */
mmio_write32(port_off, portsc);
DKM_STAT_INC(xhci_port_reset_ok);
/* Drain any Port Status Change Events from the event ring.
* Port reset generates a Port Status Change Event that must
* be consumed before submitting new commands, otherwise the
* event ring gets congested and command completion events
* may not be delivered. */
dkm_delay_ms(10); /* Allow controller to post the event */
xhci_trb *evts = (xhci_trb *)g_evt_ring.dma.virt;
for (int drain = 0; drain < 16; drain++) {
xhci_trb *evt = &evts[g_evt_ring.enqueue];
u32 ctrl = evt->control;
u8 cycle = (u8)(ctrl & TRB_CYCLE);
if (cycle != (u8)g_evt_ccs) break; /* No more events */
u8 type = (u8)((ctrl >> TRB_TYPE_SHIFT) & TRB_TYPE_MASK);
if (type == EVT_TYPE_PORT_CHANGE) {
u32 port_id = (u32)((evt->parameter >> 24) & 0xFF);
log_hex("[xhci] drained port change event, port=", port_id);
}
/* Advance dequeue */
g_evt_ring.enqueue = (g_evt_ring.enqueue + 1) % (g_evt_ring.size - 1);
if (g_evt_ring.enqueue == 0) g_evt_ccs ^= 1;
u64 erdp = g_evt_ring.dma.phys + (u64)g_evt_ring.enqueue * sizeof(xhci_trb);
mmio_write64(g_runtime_off + XHCI_RT_IR0_ERDP_LO, erdp | g_evt_ccs);
}
return 0;
}
/* Check if device disconnected during reset */
if (!(portsc & PORTSC_CCS)) return -2;
__asm__ volatile("pause");
}
g_log->warn("[xhci] port reset timeout");
return -3;
}
static u8 xhci_get_port_speed(u32 port_idx) {
u32 port_off = g_port_base + port_idx * XHCI_PORT_STRIDE + XHCI_PORT_SC;
u32 portsc = mmio_read32(port_off);
return (u8)((portsc >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK);
}
/* ================================================================
* Phase 3: Enable Slot
* ================================================================ */
static int xhci_enable_slot(u8 *out_slot_id) {
/* Submit Enable Slot Command TRB */
u32 control = (TRB_TYPE_ENABLE_SLOT << TRB_TYPE_SHIFT);
u64 out_param = 0;
u32 out_status = 0;
int rc = xhci_cmd_submit(0, 0, control, 5000, &out_param, &out_status);
if (rc != 0) {
g_log->error("[xhci] Enable Slot failed");
return rc;
}
/* Slot ID is in the upper bits of the completion event parameter */
*out_slot_id = (u8)((out_param >> 24) & 0xFF);
log_hex("[xhci] slot_id=", *out_slot_id);
return 0;
}
/* ================================================================
* Phase 3: Address Device
* ================================================================ */
static int xhci_address_device(u8 slot_id, int bsr) {
/* Input Context physical address */
u64 input_ctx_phys = g_input_ctx.phys;
u32 control = (TRB_TYPE_ADDRESS_DEV << TRB_TYPE_SHIFT);
if (bsr) control |= (1u << 9); /* BSR (Block Set Address) */
/* Slot ID goes in control bits [31:24] */
control |= ((u32)slot_id << 24);
u64 out_param = 0;
u32 out_status = 0;
int rc = xhci_cmd_submit(input_ctx_phys, 0, control, 5000, &out_param, &out_status);
if (rc != 0) {
g_log->error("[xhci] Address Device failed");
return rc;
}
return 0;
}
/* ================================================================
* Phase 3: Configure Endpoint
* ================================================================ */
static int xhci_configure_endpoint(u8 slot_id) {
u64 input_ctx_phys = g_input_ctx.phys;
u32 control = (TRB_TYPE_CONFIGURE_EP << TRB_TYPE_SHIFT) | ((u32)slot_id << 24);
u64 out_param = 0;
u32 out_status = 0;
int rc = xhci_cmd_submit(input_ctx_phys, 0, control, 5000, &out_param, &out_status);
if (rc != 0) {
g_log->error("[xhci] Configure Endpoint failed");
return rc;
}
return 0;
}
/* ================================================================
* Phase 3: Control Transfer (EP0)
* ================================================================ */
static int xhci_control_transfer(const u8 *setup_pkt, u8 *data, u32 data_len,
u8 direction) {
/* Setup Stage TRB */
u64 setup_val = ((u64)setup_pkt[4] << 32) | ((u64)setup_pkt[5] << 40) |
((u64)setup_pkt[6] << 48) | ((u64)setup_pkt[7] << 56);
setup_val |= (u64)setup_pkt[0] | ((u64)setup_pkt[1] << 8) |
((u64)setup_pkt[2] << 16) | ((u64)setup_pkt[3] << 24);
u32 setup_status = 8; /* Setup packet length */
u32 setup_control = (TRB_TYPE_SETUP_STAGE << TRB_TYPE_SHIFT) | TRB_IOC;
/* Set direction in TRB: bit 16 of Setup Stage TRB */
if (direction) setup_control |= (1u << 16);
xhci_ring_enqueue(&g_ep0_ring, setup_val, setup_status, setup_control);
/* Data Stage TRB (if data_len > 0) */
if (data_len > 0 && data) {
/* Copy data to transfer buffer for OUT direction */
if (direction == 0) {
xhci_memcpy(g_xfer_buf.virt, data, data_len);
__asm__ volatile("wbinvd" ::: "memory");
}
u64 data_phys = direction ? g_xfer_buf.phys : g_xfer_buf.phys;
u32 data_status = data_len;
u32 data_control = (TRB_TYPE_DATA_STAGE << TRB_TYPE_SHIFT) | TRB_IOC;
if (direction) data_control |= (1u << 16); /* DIR=IN */
xhci_ring_enqueue(&g_ep0_ring, data_phys, data_status, data_control);
}
/* Status Stage TRB */
u32 status_dir = direction ? 0 : 1; /* opposite of data direction */
u32 status_control = (TRB_TYPE_STATUS_STAGE << TRB_TYPE_SHIFT) | TRB_IOC;
if (status_dir) status_control |= (1u << 16);
xhci_ring_enqueue(&g_ep0_ring, 0, 0, status_control);
xhci_mb();
xhci_ring_doorbell(g_msc.slot_id, 1); /* EP1 = EP0+1 */
/* Poll for Transfer Event */
u64 timeout_ms = 5000;
u64 out_param = 0;
u32 out_status = 0;
int rc = xhci_poll_event(EVT_TYPE_TRANSFER, timeout_ms,
&out_param, &out_status, 0);
if (rc != 0) {
g_log->warn("[xhci] control transfer failed");
return rc;
}
/* For IN direction, copy data from DMA buffer */
if (direction && data_len > 0 && data) {
xhci_memcpy(data, g_xfer_buf.virt, data_len);
}
/* Clear EINT */
mmio_write32(g_cap_length + XHCI_OP_USBSTS,
mmio_read32(g_cap_length + XHCI_OP_USBSTS) | USBSTS_EINT);
return 0;
}
/* ================================================================
* Phase 3: USB Descriptor reads
* ================================================================ */
static int xhci_get_device_descriptor(u8 *out_desc) {
u8 setup[8];
setup[0] = 0x80; /* bmRequestType: Device-to-Host, Standard, Device */
setup[1] = 0x06; /* bRequest: GET_DESCRIPTOR */
setup[2] = 0x00; /* wValue low: Descriptor Index */
setup[3] = 0x01; /* wValue high: Device Descriptor */
setup[4] = 0x00; /* wIndex low */
setup[5] = 0x00; /* wIndex high */
setup[6] = 0x12; /* wLength low: 18 bytes */
setup[7] = 0x00; /* wLength high */
xhci_zero(out_desc, 18);
return xhci_control_transfer(setup, out_desc, 18, 1);
}
static int xhci_get_config_descriptor(u8 *out_desc, u32 max_len) {
u8 setup[8];
setup[0] = 0x80; /* Device-to-Host, Standard, Device */
setup[1] = 0x06; /* GET_DESCRIPTOR */
setup[2] = 0x00; /* Index */
setup[3] = 0x02; /* Configuration Descriptor */
setup[4] = 0x00;
setup[5] = 0x00;
setup[6] = (u8)(max_len & 0xFF);
setup[7] = (u8)((max_len >> 8) & 0xFF);
xhci_zero(out_desc, max_len);
return xhci_control_transfer(setup, out_desc, max_len, 1);
}
/* ================================================================
* Phase 4: Bulk Transfer
* ================================================================ */
static int xhci_bulk_transfer(xhci_ring *ring, u8 ep_num,
u8 direction, /* 0=OUT, 1=IN */
const void *data, u32 data_len) {
if (data_len == 0) return -1;
/* For OUT, copy data to DMA buffer */
if (direction == 0) {
xhci_memcpy(g_xfer_buf.virt, data, data_len);
__asm__ volatile("wbinvd" ::: "memory");
}
/* Enqueue Normal TRB(s) — single TRB for now (up to 64KB) */
u32 control = (TRB_TYPE_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC;
xhci_ring_enqueue(ring, g_xfer_buf.phys, data_len, control);
xhci_mb();
xhci_ring_doorbell(g_msc.slot_id, ep_num);
/* Poll for Transfer Event */
u64 out_param = 0;
u32 out_status = 0;
int rc = xhci_poll_event(EVT_TYPE_TRANSFER, 10000,
&out_param, &out_status, 0);
if (rc != 0) {
g_log->warn("[xhci] bulk transfer failed");
return rc;
}
/* For IN, copy data from DMA buffer */
if (direction == 1) {
xhci_memcpy((void *)data, g_xfer_buf.virt, data_len);
}
/* Clear EINT */
mmio_write32(g_cap_length + XHCI_OP_USBSTS,
mmio_read32(g_cap_length + XHCI_OP_USBSTS) | USBSTS_EINT);
return 0;
}
/* ================================================================
* Phase 4: BOT (Bulk-Only Transport) + SCSI
* ================================================================ */
static int xhci_bot_send_cbw(u8 lun, u8 flags, u32 data_len,
const u8 *cb, u8 cb_len) {
u8 cbw[31];
xhci_zero(cbw, 31);
/* Signature */
cbw[0] = 0x55; cbw[1] = 0x53; cbw[2] = 0x42; cbw[3] = 0x43;
/* Tag (little-endian) */
u32 tag = g_msc.cbw_tag++;
cbw[4] = (u8)(tag & 0xFF);
cbw[5] = (u8)((tag >> 8) & 0xFF);
cbw[6] = (u8)((tag >> 16) & 0xFF);
cbw[7] = (u8)((tag >> 24) & 0xFF);
/* Data Transfer Length (little-endian) */
cbw[8] = (u8)(data_len & 0xFF);
cbw[9] = (u8)((data_len >> 8) & 0xFF);
cbw[10] = (u8)((data_len >> 16) & 0xFF);
cbw[11] = (u8)((data_len >> 24) & 0xFF);
/* Flags */
cbw[12] = flags;
/* LUN */
cbw[13] = lun;
/* CB Length */
cbw[14] = cb_len;
/* Command Block */
for (u8 i = 0; i < cb_len && i < 16; i++)
cbw[15 + i] = cb[i];
return xhci_bulk_transfer(&g_bulk_out_ring, g_msc.bulk_out_ep,
0, cbw, 31);
}
static int xhci_bot_receive_csw(void) {
int rc = xhci_bulk_transfer(&g_bulk_in_ring, g_msc.bulk_in_ep,
1, g_csw_buf.virt, 13);
if (rc != 0) return rc;
/* Verify CSW signature */
u8 *csw = (u8 *)g_csw_buf.virt;
if (csw[0] != 0x55 || csw[1] != 0x53 || csw[2] != 0x42 || csw[3] != 0x53) {
g_log->error("[xhci] CSW signature mismatch");
return -2;
}
/* CSW status: byte 12, 0=OK */
u8 status = csw[12];
if (status != 0) {
log_hex("[xhci] CSW status=", status);
return -3;
}
return 0;
}
/* SCSI INQUIRY */
static int xhci_scsi_inquiry(void) {
u8 cb[16];
xhci_zero(cb, 16);
cb[0] = SCSI_INQUIRY;
cb[4] = 36; /* allocation length */
int rc = xhci_bot_send_cbw(0, 0x80, 36, cb, 6);
if (rc != 0) return rc;
/* Data phase: IN 36 bytes */
u8 inq_data[36];
rc = xhci_bulk_transfer(&g_bulk_in_ring, g_msc.bulk_in_ep,
1, inq_data, 36);
if (rc != 0) return rc;
/* CSW */
rc = xhci_bot_receive_csw();
if (rc != 0) return rc;
/* Parse INQUIRY data */
u8 dev_type = inq_data[0] & 0x1F;
log_hex("[xhci] INQ PeripheralType=", dev_type);
g_log->info("[xhci] INQ Vendor:");
g_log->info((const char *)(inq_data + 8)); /* Not null-terminated but safe for log */
return (dev_type == 0x00) ? 0 : -1; /* 0 = Direct Access Block */
}
/* SCSI READ CAPACITY(10) */
static int xhci_scsi_read_capacity(u64 *out_sectors, u32 *out_blk_size) {
u8 cb[16];
xhci_zero(cb, 16);
cb[0] = SCSI_READ_CAPACITY10;
int rc = xhci_bot_send_cbw(0, 0x80, 8, cb, 10);
if (rc != 0) return rc;
/* Data phase: IN 8 bytes */
u8 cap_data[8];
rc = xhci_bulk_transfer(&g_bulk_in_ring, g_msc.bulk_in_ep,
1, cap_data, 8);
if (rc != 0) return rc;
/* CSW */
rc = xhci_bot_receive_csw();
if (rc != 0) return rc;
/* Parse big-endian */
u32 max_lba = ((u32)cap_data[0] << 24) | ((u32)cap_data[1] << 16) |
((u32)cap_data[2] << 8) | (u32)cap_data[3];
u32 blk_size = ((u32)cap_data[4] << 24) | ((u32)cap_data[5] << 16) |
((u32)cap_data[6] << 8) | (u32)cap_data[7];
*out_sectors = (u64)max_lba + 1; /* READ CAPACITY returns max LBA */
*out_blk_size = blk_size;
log_hex("[xhci] capacity sectors=", *out_sectors);
log_hex("[xhci] block size=", blk_size);
return 0;
}
/* SCSI TEST UNIT READY */
static int xhci_scsi_test_unit_ready(void) {
u8 cb[16];
xhci_zero(cb, 16);
cb[0] = SCSI_TEST_UNIT_READY;
int rc = xhci_bot_send_cbw(0, 0x00, 0, cb, 6);
if (rc != 0) return rc;
return xhci_bot_receive_csw();
}
/* SCSI READ(10) */
static int xhci_scsi_read10(u64 lba, u32 count, void *buffer) {
u8 cb[16];
xhci_zero(cb, 16);
cb[0] = SCSI_READ10;
cb[2] = (u8)((lba >> 24) & 0xFF);
cb[3] = (u8)((lba >> 16) & 0xFF);
cb[4] = (u8)((lba >> 8) & 0xFF);
cb[5] = (u8)(lba & 0xFF);
cb[7] = (u8)((count >> 8) & 0xFF);
cb[8] = (u8)(count & 0xFF);
u32 data_len = count * 512;
int rc = xhci_bot_send_cbw(0, 0x80, data_len, cb, 10);
if (rc != 0) return rc;
/* Data phase: IN */
rc = xhci_bulk_transfer(&g_bulk_in_ring, g_msc.bulk_in_ep,
1, buffer, data_len);
if (rc != 0) return rc;
return xhci_bot_receive_csw();
}
/* SCSI WRITE(10) */
static int xhci_scsi_write10(u64 lba, u32 count, const void *buffer) {
u8 cb[16];
xhci_zero(cb, 16);
cb[0] = SCSI_WRITE10;
cb[2] = (u8)((lba >> 24) & 0xFF);
cb[3] = (u8)((lba >> 16) & 0xFF);
cb[4] = (u8)((lba >> 8) & 0xFF);
cb[5] = (u8)(lba & 0xFF);
cb[7] = (u8)((count >> 8) & 0xFF);
cb[8] = (u8)(count & 0xFF);
u32 data_len = count * 512;
int rc = xhci_bot_send_cbw(0, 0x00, data_len, cb, 10);
if (rc != 0) return rc;
/* Data phase: OUT */
rc = xhci_bulk_transfer(&g_bulk_out_ring, g_msc.bulk_out_ep,
0, buffer, data_len);
if (rc != 0) return rc;
return xhci_bot_receive_csw();
}
/* ================================================================
* Phase 4+5: Block device interface
* ================================================================ */
static int xhci_block_read(void *ctx, u64 lba, u32 count, void *buffer) {
(void)ctx;
if (!g_msc.ready || !buffer || count == 0) return -1;
DKM_STAT_INC(xhci_block_reads);
u8 *buf = (u8 *)buffer;
while (count > 0) {
u32 chunk = count > 16 ? 16 : count; /* max 16 sectors (8KB) per SCSI cmd */
int rc = xhci_scsi_read10(lba, chunk, g_xfer_buf.virt);
if (rc != 0) {
log_hex("[xhci] READ10 failed lba=", lba);
return rc;
}
xhci_memcpy(buf, g_xfer_buf.virt, chunk * 512);
buf += chunk * 512;
lba += chunk;
count -= chunk;
}
return 0;
}
static int xhci_block_write(void *ctx, u64 lba, u32 count, const void *buffer) {
(void)ctx;
if (!g_msc.ready || !buffer || count == 0) return -1;
DKM_STAT_INC(xhci_block_writes);
const u8 *buf = (const u8 *)buffer;
while (count > 0) {
u32 chunk = count > 16 ? 16 : count;
xhci_memcpy(g_xfer_buf.virt, buf, chunk * 512);
__asm__ volatile("wbinvd" ::: "memory"); /* Cache flush for DMA */
int rc = xhci_scsi_write10(lba, chunk, g_xfer_buf.virt);
if (rc != 0) {
log_hex("[xhci] WRITE10 failed lba=", lba);
return rc;
}
buf += chunk * 512;
lba += chunk;
count -= chunk;
}
return 0;
}
/* ================================================================
* Phase 3: Build Input Context for Address Device
* ================================================================ */
static void xhci_build_input_ctx_slot(u8 slot_id, u8 speed, u16 mps0) {
(void)slot_id;
u32 ctx_sz = g_ctx_size;
u8 *ic = (u8 *)g_input_ctx.virt;
xhci_zero(ic, g_input_ctx.size);
/* Input Control Context: enable Slot Context (bit 0) and EP0 (bit 1) */
u32 *icc = (u32 *)ic;
icc[0] = (1u << 1) | (1u << 0); /* A0=1 (Slot), A1=1 (EP0) */
/* Slot Context starts at offset ctx_sz */
u32 *slot_ctx = (u32 *)(ic + ctx_sz);
u32 route_string = 0; /* direct-connect, no hub */
slot_ctx[0] = route_string | ((u32)speed << 29);
slot_ctx[1] = (u32)mps0; /* Max Packet Size for EP0 */
slot_ctx[2] = (1u << 27); /* Context Entries = 1 (only EP0) */
/* EP0 Context starts at offset ctx_sz * 2 */
u32 *ep0_ctx = (u32 *)(ic + ctx_sz * 2);
/* EP Type = Control, Max Packet Size, etc. */
ep0_ctx[1] = (4u << 3) | ((u32)mps0 << 16); /* EP Type=4 (Control IN), MPS */
ep0_ctx[4] = 8; /* Average TRB Length */
/* Dequeue Pointer: EP0 transfer ring */
u64 ep0_deq = g_ep0_ring.dma.phys | (u64)g_ep0_ring.ccs;
ep0_ctx[6] = (u32)(ep0_deq & 0xFFFFFFFFu);
ep0_ctx[7] = (u32)(ep0_deq >> 32);
}
static void xhci_build_input_ctx_bulk_eps(u8 bulk_out_ep, u8 bulk_in_ep,
u16 out_mps, u16 in_mps) {
u32 ctx_sz = g_ctx_size;
u8 *ic = (u8 *)g_input_ctx.virt;
/* Update Input Control Context: enable EP0, Bulk OUT, Bulk IN */
u32 *icc = (u32 *)ic;
icc[0] = (1u << 1) | (1u << 0) |
(1u << bulk_out_ep) | (1u << bulk_in_ep);
/* Update Slot Context: Context Entries = max(bulk_in_ep, bulk_out_ep) */
u32 *slot_ctx = (u32 *)(ic + ctx_sz);
u32 max_ep = bulk_out_ep > bulk_in_ep ? bulk_out_ep : bulk_in_ep;
slot_ctx[2] = (max_ep << 27);
/* Bulk OUT Endpoint Context (at offset ctx_sz * (1 + bulk_out_ep)) */
u32 *out_ctx = (u32 *)(ic + ctx_sz * (1 + bulk_out_ep));
out_ctx[1] = (EP_TYPE_BULK_OUT << 3) | ((u32)out_mps << 16);
out_ctx[4] = 512; /* Average TRB Length */
u64 out_deq = g_bulk_out_ring.dma.phys | (u64)g_bulk_out_ring.ccs;
out_ctx[6] = (u32)(out_deq & 0xFFFFFFFFu);
out_ctx[7] = (u32)(out_deq >> 32);
/* Bulk IN Endpoint Context (at offset ctx_sz * (1 + bulk_in_ep)) */
u32 *in_ctx = (u32 *)(ic + ctx_sz * (1 + bulk_in_ep));
in_ctx[1] = (EP_TYPE_BULK_IN << 3) | ((u32)in_mps << 16);
in_ctx[4] = 512; /* Average TRB Length */
u64 in_deq = g_bulk_in_ring.dma.phys | (u64)g_bulk_in_ring.ccs;
in_ctx[6] = (u32)(in_deq & 0xFFFFFFFFu);
in_ctx[7] = (u32)(in_deq >> 32);
}
/* ================================================================
* Phase 3: Full device enumeration
* ================================================================ */
static int xhci_enum_msc_device(u32 port_idx) {
u8 speed = xhci_get_port_speed(port_idx);
log_hex("[xhci] port speed=", speed);
/* Default MaxPacketSize for EP0 based on speed */
u16 mps0 = 64; /* High/Super-Speed default */
if (speed == USB_SPEED_FULL) mps0 = 64;
if (speed == USB_SPEED_LOW) mps0 = 8;
/* Step 1: Enable Slot */
u8 slot_id = 0;
if (xhci_enable_slot(&slot_id) != 0) return -1;
g_msc.slot_id = slot_id;
g_msc.speed = speed;
/* Step 2: Allocate per-device DMA buffers */
if (g_api->dma->alloc_pages(4, 4096, 0x100000000ULL, &g_input_ctx) != 0) return -2;
if (g_api->dma->alloc_pages(4, 4096, 0x100000000ULL, &g_dev_ctx) != 0) return -2;
if (g_api->dma->alloc_pages(4, 4096, 0x100000000ULL, &g_xfer_buf) != 0) return -2;
if (g_api->dma->alloc_pages(1, 4096, 0x100000000ULL, &g_csw_buf) != 0) return -2;
if (xhci_ring_init(&g_ep0_ring, XFER_RING_SIZE) != 0) return -2;
if (xhci_ring_init(&g_bulk_out_ring, XFER_RING_SIZE) != 0) return -2;
if (xhci_ring_init(&g_bulk_in_ring, XFER_RING_SIZE) != 0) return -2;
/* Step 3: Build Input Context and Address Device (BSR=1 first) */
xhci_build_input_ctx_slot(slot_id, speed, mps0);
if (xhci_address_device(slot_id, 1) != 0) {
/* BSR=1 failed, try without BSR */
xhci_build_input_ctx_slot(slot_id, speed, mps0);
if (xhci_address_device(slot_id, 0) != 0) return -3;
}
/* Step 4: Get Device Descriptor */
u8 dev_desc[18];
if (xhci_get_device_descriptor(dev_desc) != 0) {
g_log->warn("[xhci] GET_DESCRIPTOR(Device) failed");
return -4;
}
u8 dev_class = dev_desc[4];
u16 vid = (u16)dev_desc[8] | ((u16)dev_desc[9] << 8);
u16 pid = (u16)dev_desc[10] | ((u16)dev_desc[11] << 8);
u8 real_mps0 = dev_desc[7];
log_hex("[xhci] dev class=", dev_class);
log_hex("[xhci] VID=", vid);
log_hex("[xhci] PID=", pid);
log_hex("[xhci] bMaxPacketSize0=", real_mps0);
/* Step 5: Re-address with real MPS if different */
if (real_mps0 != mps0 && real_mps0 >= 8) {
mps0 = real_mps0;
xhci_build_input_ctx_slot(slot_id, speed, mps0);
if (xhci_address_device(slot_id, 0) != 0) {
g_log->warn("[xhci] re-address failed");
/* Continue anyway — device may still work */
}
}
/* Step 6: Get Configuration Descriptor */
u8 cfg_desc[255];
if (xhci_get_config_descriptor(cfg_desc, 255) != 0) {
g_log->warn("[xhci] GET_DESCRIPTOR(Config) failed");
return -5;
}
/* Parse configuration descriptor to find MSC interface and endpoints */
u16 total_len = (u16)cfg_desc[2] | ((u16)cfg_desc[3] << 8);
if (total_len < 9) return -5;
u8 bulk_out_ep = 0, bulk_in_ep = 0;
u16 bulk_out_mps = 512, bulk_in_mps = 512;
int msc_found = 0;
/* Walk through descriptors: config(9) + interface(9) + endpoints(7*N) */
u32 pos = 9; /* skip config descriptor */
while (pos + 2 <= (u32)total_len && pos + cfg_desc[pos] <= (u32)total_len) {
u8 desc_len = cfg_desc[pos];
u8 desc_type = cfg_desc[pos + 1];
if (desc_type == 4 && desc_len >= 9) {
/* Interface Descriptor */
u8 iface_class = cfg_desc[pos + 5];
u8 iface_subclass = cfg_desc[pos + 6];
u8 iface_protocol = cfg_desc[pos + 7];
if (iface_class == 0x08 && iface_subclass == 0x06 && iface_protocol == 0x50) {
msc_found = 1;
g_log->info("[xhci] MSC BOT interface found");
/* Scan following endpoint descriptors */
u32 ep_pos = pos + desc_len;
while (ep_pos + 7 <= (u32)total_len &&
cfg_desc[ep_pos + 1] == 5) { /* endpoint descriptor */
u8 ep_addr = cfg_desc[ep_pos + 2];
u8 ep_attr = cfg_desc[ep_pos + 3];
u16 ep_mps = (u16)cfg_desc[ep_pos + 4] | ((u16)cfg_desc[ep_pos + 5] << 8);
if ((ep_attr & 0x03) == 0x02) { /* Bulk */
if (ep_addr & 0x80) {
bulk_in_ep = ep_addr & 0x0F;
bulk_in_mps = ep_mps;
} else {
bulk_out_ep = ep_addr & 0x0F;
bulk_out_mps = ep_mps;
}
}
ep_pos += cfg_desc[ep_pos];
}
break;
}
}
pos += desc_len;
}
if (!msc_found || bulk_out_ep == 0 || bulk_in_ep == 0) {
g_log->warn("[xhci] no MSC BOT interface found");
return -6;
}
log_hex("[xhci] Bulk OUT EP=", bulk_out_ep);
log_hex("[xhci] Bulk IN EP=", bulk_in_ep);
log_hex("[xhci] Bulk OUT MPS=", bulk_out_mps);
log_hex("[xhci] Bulk IN MPS=", bulk_in_mps);
g_msc.bulk_out_ep = bulk_out_ep;
g_msc.bulk_in_ep = bulk_in_ep;
g_msc.bulk_out_mps = bulk_out_mps;
g_msc.bulk_in_mps = bulk_in_mps;
/* Step 7: Configure Endpoint */
xhci_zero(g_input_ctx.virt, g_input_ctx.size);
xhci_build_input_ctx_bulk_eps(bulk_out_ep, bulk_in_ep,
bulk_out_mps, bulk_in_mps);
if (xhci_configure_endpoint(slot_id) != 0) {
g_log->warn("[xhci] Configure Endpoint failed");
return -7;
}
/* Step 8: SCSI INQUIRY to verify MSC device */
if (xhci_scsi_inquiry() != 0) {
g_log->warn("[xhci] INQUIRY failed or not block device");
/* Continue anyway — some devices return non-zero peripheral type */
}
/* Step 9: TEST UNIT READY (retry up to 5 times) */
for (int i = 0; i < 5; i++) {
if (xhci_scsi_test_unit_ready() == 0) break;
dkm_delay_ms(100);
}
/* Step 10: READ CAPACITY */
u64 sectors = 0;
u32 blk_size = 512;
if (xhci_scsi_read_capacity(&sectors, &blk_size) != 0) {
g_log->warn("[xhci] READ CAPACITY failed");
/* Use defaults */
sectors = 0;
blk_size = 512;
}
g_msc.sector_count = sectors;
g_msc.sector_size = blk_size;
g_msc.cbw_tag = 1;
g_msc.ready = 1;
DKM_STAT_INC(xhci_msc_found);
log_hex("[xhci] MSC device ready, sectors=", sectors);
log_hex("[xhci] MSC device block_size=", blk_size);
return 0;
}
/* ================================================================
* driver_init — main entry point
* ================================================================ */
__attribute__((visibility("default")))
int driver_init(const struct dkm_kernel_api *api,
struct dkm_driver_handle *handle) {
(void)handle;
dkm_tsc_calibrate();
if (!api || !api->log) return -1;
g_api = api;
g_log = api->log;
dkm_instr_init(api);
g_log->info("[xhci] init begin");
/* ---- Phase 1: PCI discovery ---- */
u8 bus = 0, dev = 0, func = 0;
if (xhci_find(&bus, &dev, &func) != 0) {
g_log->info("[xhci] xHCI controller not found");
g_log->info("[xhci] driver ready");
return 0;
}
g_pci_bus = bus; g_pci_dev = dev; g_pci_func = func;
DKM_STAT_INC(xhci_pci_found);
g_log->info("[xhci] xHCI controller found");
log_hex("[xhci] bus=", bus);
log_hex("[xhci] dev=", dev);
log_hex("[xhci] func=", func);
u32 vd = dkm_pci_read(bus, dev, func, PCI_VENDOR_ID);
log_hex("[xhci] vendor=", vd & 0xffff);
log_hex("[xhci] device=", vd >> 16);
/* Enable PCI MEM + BUSM, disable INTx (polling mode) */
u32 command = dkm_pci_read(bus, dev, func, PCI_COMMAND) & 0xffff;
command |= PCI_CMD_MEM | PCI_CMD_BUSM | (1u << 10);
dkm_pci_write(bus, dev, func, PCI_COMMAND, command);
log_hex("[xhci] PCI command=", command);
/* Read BAR0 (64-bit) */
u32 bar0_lo = dkm_pci_read(bus, dev, func, PCI_BAR0);
u32 bar1_hi = dkm_pci_read(bus, dev, func, PCI_BAR1);
u64 bar_phys;
if (bar0_lo & 1u) {
g_log->warn("[xhci] BAR0 is IO space; unsupported");
g_log->info("[xhci] driver ready");
return 0;
}
if ((bar0_lo & 0x6u) == 0x4u) {
bar_phys = ((u64)bar1_hi << 32) | (u64)(bar0_lo & 0xFFFFFFF0u);
} else {
bar_phys = (u64)(bar0_lo & 0xFFFFFFF0u);
}
u32 irq_line = dkm_pci_read(bus, dev, func, PCI_IRQ_LINE) & 0xff;
log_hex("[xhci] BAR0 phys=", bar_phys);
log_hex("[xhci] IRQ line=", irq_line);
log_hex("[xhci] HHDM offset=", api->hhdm_offset);
if (!api->hhdm_offset || !bar_phys) {
g_log->warn("[xhci] missing HHDM or BAR0; skip");
g_log->info("[xhci] driver ready");
return 0;
}
/* ---- Phase 1: MMIO mapping ---- */
g_mmio = xhci_map_bar(api, bar_phys);
if (!g_mmio) {
g_log->error("[xhci] BAR MMIO unavailable; driver inactive");
g_log->info("[xhci] driver ready");
return 0;
}
/* ---- Phase 1: Parse capabilities ---- */
if (xhci_parse_caps() != 0) {
g_log->error("[xhci] capability parse failed");
g_log->info("[xhci] driver ready");
return 0;
}
/* ---- Phase 1: BIOS Handoff ---- */
xhci_bios_handoff();
/* ---- Phase 1: Controller Reset ---- */
if (xhci_controller_reset() != 0) {
g_log->error("[xhci] controller reset failed");
g_log->info("[xhci] driver ready");
return 0;
}
/* ---- Phase 2: Setup Rings + DCBAA + Scratchpad ---- */
if (!api->dma) {
g_log->warn("[xhci] DMA API unavailable");
g_log->info("[xhci] driver ready");
return 0;
}
if (xhci_setup_rings() != 0) {
g_log->error("[xhci] ring setup failed");
g_log->info("[xhci] driver ready");
return 0;
}
/* ---- Phase 2: Start Controller ---- */
if (xhci_controller_start() != 0) {
g_log->error("[xhci] controller start failed");
g_log->info("[xhci] driver ready");
return 0;
}
/* ---- Phase 3: Port Scan + Device Enumeration ---- */
/*
* Wait 100ms after controller start for QEMU to complete USB device
* attachment and update port status. Without this delay, PORTSC.CCS
* may read 0 even though a device is physically connected.
*/
g_log->info("[xhci] waiting 100ms for port settle...");
dkm_delay_ms(100);
g_msc.ready = 0;
for (u32 port = 0; port < g_max_ports; port++) {
u32 port_off = g_port_base + port * XHCI_PORT_STRIDE + XHCI_PORT_SC;
u32 portsc = mmio_read32(port_off);
/* Always log PORTSC for debugging, even if no device is connected */
log_hex("[xhci] port=", port);
log_hex("[xhci] PORTSC=", portsc);
if (!(portsc & PORTSC_CCS)) {
/*
* If PPC (Per-Port Power Control) is supported, the port may
* need to be powered on before CCS reflects device connection.
* Set PP (Port Power) bit in PORTSC to enable port power.
* If PPC=0, ports are always powered and this write is a no-op.
*/
if (g_ppc && !(portsc & (1u << 9))) {
g_log->info("[xhci] enabling port power");
mmio_write32(port_off, portsc | (1u << 9));
/* Wait briefly for power to stabilize */
dkm_delay_ms(20);
portsc = mmio_read32(port_off);
log_hex("[xhci] PORTSC after PP=", portsc);
}
if (!(portsc & PORTSC_CCS)) continue; /* Still no device */
}
g_log->info("[xhci] device connected");
/* Port Reset */
if (xhci_reset_port(port) != 0) {
g_log->warn("[xhci] port reset failed; skipping");
continue;
}
g_log->info("[xhci] port reset OK");
/* Try to enumerate as MSC device */
if (xhci_enum_msc_device(port) == 0) {
g_log->info("[xhci] MSC enumeration successful");
break; /* First MSC device is enough for now */
}
g_log->warn("[xhci] MSC enumeration failed; trying next port");
}
/* ---- Phase 5: Register Block Device ---- */
if (g_msc.ready) {
if (api->block && api->block->register_device) {
struct dkm_block_device_desc desc;
desc.name = "usb0";
desc.sector_size = g_msc.sector_size;
desc.sector_count = g_msc.sector_count;
desc.ctx = 0;
desc.read = xhci_block_read;
desc.write = xhci_block_write;
int index = api->block->register_device(&desc);
log_hex("[xhci] block provider index=", (u64)(i64)index);
} else {
g_log->warn("[xhci] block API unavailable; provider not registered");
}
} else {
g_log->info("[xhci] no MSC device found; block provider not registered");
}
g_log->info("[xhci] driver ready");
return 0;
}
__attribute__((visibility("default")))
int driver_exit(struct dkm_driver_handle *handle) {
(void)handle;
/* Halt controller if it was started */
if (g_mmio) {
u32 usbcmd = mmio_read32(g_cap_length + XHCI_OP_USBCMD);
mmio_write32(g_cap_length + XHCI_OP_USBCMD, usbcmd & ~USBCMD_RS);
}
return 0;
}