713 lines
25 KiB
C
713 lines
25 KiB
C
/* DKM AHCI Driver — PCI AHCI discovery and ABAR read-only enumeration
|
||
* Stage 1 optional storage driver, depends on "pci" and "irq", provides "block".
|
||
* This first version does not issue DMA commands; it only discovers the HBA
|
||
* and ports so later block I/O can be added safely.
|
||
*/
|
||
|
||
#include "../dkm_shared.h"
|
||
#include "../dkm_instr.h"
|
||
|
||
DKM_STAT_DECL(ahci_port_found);
|
||
DKM_STAT_DECL(ahci_identify_ok);
|
||
DKM_STAT_DECL(ahci_block_reads);
|
||
DKM_STAT_DECL(ahci_block_writes);
|
||
DKM_TS_DECL(ts_abar);
|
||
|
||
static const char *const g_depends[] = { "pci", "irq" };
|
||
static const char *const g_provides[] = { "block" };
|
||
|
||
__attribute__((visibility("default"), used))
|
||
const struct dkm_driver_desc driver_desc = {
|
||
.magic = DKM_DRIVER_MAGIC,
|
||
.abi_version = DKM_ABI_VERSION,
|
||
.desc_size = sizeof(struct dkm_driver_desc),
|
||
.name = "ahci",
|
||
.version = "0.1.0",
|
||
.vendor = "Deshab",
|
||
.driver_class = 6, /* DKM_CLASS_STORAGE */
|
||
.stage = 1,
|
||
.flags = 0, /* optional */
|
||
.priority = 0,
|
||
.depends = g_depends,
|
||
.depends_count = 2,
|
||
.provides = g_provides,
|
||
.provides_count = 1,
|
||
.min_kernel_abi = 1,
|
||
.feature_bits = 0,
|
||
.reserved0 = 0,
|
||
.reserved1 = 0
|
||
};
|
||
|
||
#define AHCI_CLASS_STORAGE 0x01
|
||
#define AHCI_SUBCLASS_SATA 0x06
|
||
#define AHCI_PROGIF_AHCI 0x01
|
||
|
||
#define HBA_CAP 0x00
|
||
#define HBA_GHC 0x04
|
||
#define HBA_IS 0x08
|
||
#define HBA_PI 0x0C
|
||
#define HBA_VS 0x10
|
||
#define HBA_CAP2 0x24
|
||
#define HBA_BOHC 0x28
|
||
#define HBA_PORT_BASE 0x100
|
||
#define HBA_PORT_SIZE 0x80
|
||
|
||
#define PxCLB 0x00
|
||
#define PxFB 0x08
|
||
#define PxIS 0x10
|
||
#define PxIE 0x14
|
||
#define PxCMD 0x18
|
||
#define PxTFD 0x20
|
||
#define PxSIG 0x24
|
||
#define PxSSTS 0x28
|
||
#define PxSCTL 0x2C
|
||
#define PxSERR 0x30
|
||
#define PxSACT 0x34
|
||
#define PxCI 0x38
|
||
#define PxSNTF 0x3C
|
||
#define PxFBS 0x40
|
||
|
||
#define AHCI_CMD_ST (1u << 0)
|
||
#define AHCI_CMD_FRE (1u << 4)
|
||
#define AHCI_CMD_W (1u << 6)
|
||
#define AHCI_CMD_FR (1u << 14)
|
||
#define AHCI_CMD_CR (1u << 15)
|
||
#define AHCI_CMD_ICC_ACTIVE (1u << 28)
|
||
|
||
#define ATA_CMD_IDENTIFY 0xEC
|
||
#define ATA_CMD_IDENTIFY_PACKET 0xA1
|
||
#define ATA_CMD_READ_DMA_EXT 0x25
|
||
#define ATA_CMD_WRITE_DMA_EXT 0x35
|
||
|
||
typedef struct __attribute__((packed)) ahci_cmd_header {
|
||
u16 flags;
|
||
u16 prdtl;
|
||
u32 prdbc;
|
||
u32 ctba;
|
||
u32 ctbau;
|
||
u32 reserved[4];
|
||
} ahci_cmd_header;
|
||
|
||
typedef struct __attribute__((packed)) ahci_prdt_entry {
|
||
u32 dba;
|
||
u32 dbau;
|
||
u32 reserved;
|
||
u32 dbc_i;
|
||
} ahci_prdt_entry;
|
||
|
||
typedef struct __attribute__((packed)) ahci_cmd_table {
|
||
u8 cfis[64];
|
||
u8 acmd[16];
|
||
u8 reserved[48];
|
||
ahci_prdt_entry prdt[1];
|
||
} ahci_cmd_table;
|
||
|
||
static const struct dkm_log_api *g_log;
|
||
static volatile u32 *g_abar;
|
||
static u32 g_ready_port = 0xffffffffu;
|
||
static ahci_cmd_header *g_cmd_header;
|
||
static struct dkm_dma_buffer g_cmd_table;
|
||
static struct dkm_dma_buffer g_data;
|
||
/* IDENTIFY 读出的用户可寻址扇区数；LBA48 优先，回退 LBA28 */
|
||
static u64 g_sector_count;
|
||
|
||
static u32 pci_read(u8 bus, u8 dev, u8 func, u8 reg) {
|
||
return dkm_pci_read(bus, dev, func, reg);
|
||
}
|
||
|
||
static void pci_write(u8 bus, u8 dev, u8 func, u8 reg, u32 value) {
|
||
dkm_pci_write(bus, dev, func, reg, value);
|
||
}
|
||
|
||
static void log_hex(const char *prefix, u64 value) {
|
||
static const char hex[] = "0123456789abcdef";
|
||
char buf[19];
|
||
u32 pos = 0;
|
||
buf[pos++] = '0';
|
||
buf[pos++] = 'x';
|
||
for (int i = 15; i >= 0; i--) {
|
||
buf[pos++] = hex[(value >> (i * 4)) & 0xf];
|
||
}
|
||
buf[pos] = 0;
|
||
g_log->info(prefix);
|
||
g_log->info(buf);
|
||
}
|
||
|
||
static void ahci_zero(void *ptr, u32 len) {
|
||
u8 *p = (u8 *)ptr;
|
||
for (u32 i = 0; i < len; i++) p[i] = 0;
|
||
}
|
||
|
||
static int ahci_find(u8 *out_bus, u8 *out_dev, u8 *out_func) {
|
||
/* PCI 规范允许 256 条 bus (0-255)，实机 AHCI 可能位于 bus > 7 */
|
||
for (u16 bus = 0; bus < 256; bus++) {
|
||
for (u8 dev = 0; dev < 32; dev++) {
|
||
u32 vd = pci_read(bus, dev, 0, PCI_VENDOR_ID);
|
||
if ((vd & 0xffff) == 0xffff) continue;
|
||
|
||
u8 header = (u8)(pci_read(bus, dev, 0, PCI_HEADER) >> 16);
|
||
u8 func_count = (header & 0x80) ? 8 : 1;
|
||
for (u8 func = 0; func < func_count; func++) {
|
||
u32 vd2 = pci_read(bus, dev, func, PCI_VENDOR_ID);
|
||
if ((vd2 & 0xffff) == 0xffff) continue;
|
||
|
||
u32 class_reg = pci_read(bus, dev, func, 0x08);
|
||
u8 prog_if = (u8)((class_reg >> 8) & 0xff);
|
||
u8 subclass = (u8)((class_reg >> 16) & 0xff);
|
||
u8 class_code = (u8)((class_reg >> 24) & 0xff);
|
||
|
||
if (class_code == AHCI_CLASS_STORAGE &&
|
||
subclass == AHCI_SUBCLASS_SATA &&
|
||
prog_if == AHCI_PROGIF_AHCI) {
|
||
*out_bus = bus;
|
||
*out_dev = dev;
|
||
*out_func = func;
|
||
return 0;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return -1;
|
||
}
|
||
|
||
static const char *sata_sig_name(u32 sig) {
|
||
if (sig == 0x00000101u) return "SATA drive";
|
||
if (sig == 0xEB140101u) return "ATAPI device";
|
||
if (sig == 0xC33C0101u) return "Enclosure mgmt";
|
||
if (sig == 0x96690101u) return "Port multiplier";
|
||
if (sig == 0x00000000u) return "empty/unknown";
|
||
return "unknown signature";
|
||
}
|
||
|
||
static void ahci_log_port(u32 port) {
|
||
if (!g_abar || port >= 32) {
|
||
return;
|
||
}
|
||
u32 base = HBA_PORT_BASE + port * HBA_PORT_SIZE;
|
||
u32 ssts = g_abar[(base + PxSSTS) / 4];
|
||
u32 sig = g_abar[(base + PxSIG) / 4];
|
||
u32 cmd = g_abar[(base + PxCMD) / 4];
|
||
u32 tfd = g_abar[(base + PxTFD) / 4];
|
||
u32 serr = g_abar[(base + PxSERR) / 4];
|
||
u32 ci = g_abar[(base + PxCI) / 4];
|
||
u32 sact = g_abar[(base + PxSACT) / 4];
|
||
|
||
u32 det = ssts & 0xf;
|
||
u32 spd = (ssts >> 4) & 0xf;
|
||
u32 ipm = (ssts >> 8) & 0xf;
|
||
|
||
log_hex("[ahci] port=", port);
|
||
log_hex("[ahci] SSTS=", ssts);
|
||
log_hex("[ahci] DET=", det);
|
||
log_hex("[ahci] SPD=", spd);
|
||
log_hex("[ahci] IPM=", ipm);
|
||
log_hex("[ahci] SIG=", sig);
|
||
g_log->info("[ahci] type");
|
||
g_log->info(sata_sig_name(sig));
|
||
log_hex("[ahci] CMD=", cmd);
|
||
log_hex("[ahci] TFD=", tfd);
|
||
log_hex("[ahci] SERR=", serr);
|
||
log_hex("[ahci] CI=", ci);
|
||
log_hex("[ahci] SACT=", sact);
|
||
}
|
||
|
||
static int ahci_stop_port(u32 port) {
|
||
u32 base = HBA_PORT_BASE + port * HBA_PORT_SIZE;
|
||
u32 cmd = g_abar[(base + PxCMD) / 4];
|
||
cmd &= ~AHCI_CMD_ST;
|
||
g_abar[(base + PxCMD) / 4] = cmd;
|
||
/* 实机: 500ms TSC 超时等待 CR 清除 */
|
||
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 500;
|
||
while (dkm_rdtsc() < deadline) {
|
||
cmd = g_abar[(base + PxCMD) / 4];
|
||
if (!(cmd & AHCI_CMD_CR)) break;
|
||
__asm__ volatile("pause");
|
||
}
|
||
cmd &= ~AHCI_CMD_FRE;
|
||
g_abar[(base + PxCMD) / 4] = cmd;
|
||
/* 实机: 500ms TSC 超时等待 FR 清除 */
|
||
deadline = dkm_rdtsc() + dkm_tsc_per_ms * 500;
|
||
while (dkm_rdtsc() < deadline) {
|
||
cmd = g_abar[(base + PxCMD) / 4];
|
||
if (!(cmd & AHCI_CMD_FR)) return 0;
|
||
__asm__ volatile("pause");
|
||
}
|
||
return -1;
|
||
}
|
||
|
||
static int ahci_start_port(u32 port) {
|
||
u32 base = HBA_PORT_BASE + port * HBA_PORT_SIZE;
|
||
u32 cmd = g_abar[(base + PxCMD) / 4];
|
||
cmd |= AHCI_CMD_FRE;
|
||
g_abar[(base + PxCMD) / 4] = cmd;
|
||
cmd |= AHCI_CMD_ST;
|
||
g_abar[(base + PxCMD) / 4] = cmd;
|
||
return 0;
|
||
}
|
||
|
||
static void ahci_fill_identify_fis(u8 *fis, u8 command) {
|
||
for (u32 i = 0; i < 64; i++) fis[i] = 0;
|
||
fis[0] = 0x27; /* Register H2D FIS */
|
||
fis[1] = 1u << 7; /* command */
|
||
fis[2] = command;
|
||
fis[7] = 1u << 6; /* LBA mode */
|
||
}
|
||
|
||
static void ahci_fill_read_fis(u8 *fis, u64 lba, u32 count) {
|
||
ahci_zero(fis, 64);
|
||
fis[0] = 0x27;
|
||
fis[1] = 1u << 7;
|
||
fis[2] = ATA_CMD_READ_DMA_EXT;
|
||
fis[4] = (u8)(lba & 0xff);
|
||
fis[5] = (u8)((lba >> 8) & 0xff);
|
||
fis[6] = (u8)((lba >> 16) & 0xff);
|
||
fis[7] = (1u << 6) | (u8)((lba >> 24) & 0x0f);
|
||
fis[8] = (u8)((lba >> 24) & 0xff);
|
||
fis[9] = (u8)((lba >> 32) & 0xff);
|
||
fis[10] = (u8)((lba >> 40) & 0xff);
|
||
fis[12] = (u8)(count & 0xff);
|
||
fis[13] = (u8)((count >> 8) & 0xff);
|
||
}
|
||
|
||
static void ahci_fill_write_fis(u8 *fis, u64 lba, u32 count) {
|
||
ahci_zero(fis, 64);
|
||
fis[0] = 0x27;
|
||
fis[1] = 1u << 7;
|
||
fis[2] = ATA_CMD_WRITE_DMA_EXT;
|
||
fis[4] = (u8)(lba & 0xff);
|
||
fis[5] = (u8)((lba >> 8) & 0xff);
|
||
fis[6] = (u8)((lba >> 16) & 0xff);
|
||
fis[7] = (1u << 6) | (u8)((lba >> 24) & 0x0f);
|
||
fis[8] = (u8)((lba >> 24) & 0xff);
|
||
fis[9] = (u8)((lba >> 32) & 0xff);
|
||
fis[10] = (u8)((lba >> 40) & 0xff);
|
||
fis[12] = (u8)(count & 0xff);
|
||
fis[13] = (u8)((count >> 8) & 0xff);
|
||
}
|
||
|
||
static int ahci_read_blocks_internal(u32 port, ahci_cmd_header *hdr, struct dkm_dma_buffer *table, struct dkm_dma_buffer *data, u64 lba, u32 count, void *buffer) {
|
||
if (!buffer || count == 0) return -1;
|
||
if (count > 8) return -2;
|
||
u32 base = HBA_PORT_BASE + port * HBA_PORT_SIZE;
|
||
ahci_cmd_table *tbl = (ahci_cmd_table *)table->virt;
|
||
u32 bytes = count * 512;
|
||
ahci_zero(tbl, sizeof(ahci_cmd_table));
|
||
ahci_zero(data->virt, bytes);
|
||
|
||
hdr[0].flags = 5;
|
||
hdr[0].prdtl = 1;
|
||
hdr[0].prdbc = 0;
|
||
hdr[0].ctba = (u32)(table->phys & 0xffffffffu);
|
||
hdr[0].ctbau = (u32)(table->phys >> 32);
|
||
|
||
tbl->prdt[0].dba = (u32)(data->phys & 0xffffffffu);
|
||
tbl->prdt[0].dbau = (u32)(data->phys >> 32);
|
||
tbl->prdt[0].dbc_i = (bytes - 1) | (1u << 31);
|
||
ahci_fill_read_fis(tbl->cfis, lba, count);
|
||
|
||
g_abar[(base + PxIS) / 4] = 0xffffffffu;
|
||
g_abar[HBA_IS / 4] = (1u << port);
|
||
g_abar[(base + PxCI) / 4] = 1;
|
||
/* TSC deadline 超时: 5 秒，实机机械 HDD 首次寻道需数百 ms */
|
||
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 5000;
|
||
while (dkm_rdtsc() < deadline) {
|
||
if ((g_abar[(base + PxCI) / 4] & 1u) == 0) {
|
||
u8 *dst = (u8 *)buffer;
|
||
u8 *src = (u8 *)data->virt;
|
||
for (u32 j = 0; j < bytes; j++) dst[j] = src[j];
|
||
return 0;
|
||
}
|
||
__asm__ volatile("pause");
|
||
}
|
||
g_log->warn("[ahci] READ timeout");
|
||
log_hex("[ahci] READ lba=", lba);
|
||
log_hex("[ahci] READ count=", count);
|
||
log_hex("[ahci] READ CI=", g_abar[(base + PxCI) / 4]);
|
||
log_hex("[ahci] READ IS=", g_abar[(base + PxIS) / 4]);
|
||
log_hex("[ahci] READ TFD=", g_abar[(base + PxTFD) / 4]);
|
||
return -3;
|
||
}
|
||
|
||
static int ahci_write_blocks_internal(u32 port, ahci_cmd_header *hdr, struct dkm_dma_buffer *table, struct dkm_dma_buffer *data, u64 lba, u32 count, const void *buffer) {
|
||
if (!buffer || count == 0) return -1;
|
||
if (count > 8) return -2;
|
||
u32 base = HBA_PORT_BASE + port * HBA_PORT_SIZE;
|
||
ahci_cmd_table *tbl = (ahci_cmd_table *)table->virt;
|
||
u32 bytes = count * 512;
|
||
ahci_zero(tbl, sizeof(ahci_cmd_table));
|
||
|
||
/* copy data from user buffer into DMA buffer */
|
||
{
|
||
u8 *dst = (u8 *)data->virt;
|
||
const u8 *src = (const u8 *)buffer;
|
||
for (u32 j = 0; j < bytes; j++) dst[j] = src[j];
|
||
}
|
||
|
||
/* BUG-P0-2: 写入前必须 flush CPU cache，否则 DMA 控制器从物理内存
|
||
* 读到陈旧数据（CPU write-back cache 尚未写回）。实机必现。 */
|
||
__asm__ volatile("wbinvd" ::: "memory");
|
||
|
||
hdr[0].flags = 5 | (1u << 6); /* CFL=5, W=1 (write) */
|
||
hdr[0].prdtl = 1;
|
||
hdr[0].prdbc = 0;
|
||
hdr[0].ctba = (u32)(table->phys & 0xffffffffu);
|
||
hdr[0].ctbau = (u32)(table->phys >> 32);
|
||
|
||
tbl->prdt[0].dba = (u32)(data->phys & 0xffffffffu);
|
||
tbl->prdt[0].dbau = (u32)(data->phys >> 32);
|
||
tbl->prdt[0].dbc_i = (bytes - 1) | (1u << 31);
|
||
ahci_fill_write_fis(tbl->cfis, lba, count);
|
||
|
||
/* P0 修复: 移除 PxCMD.W 写入。AHCI 规范中 PxCMD.W 是只读位，
|
||
* 由硬件根据命令头 flags.W 自动设置。软件写入此位违反规范，
|
||
* 可能在某些实机 AHCI 控制器上导致未定义行为。 */
|
||
/* (原代码: cmd |= AHCI_CMD_W; g_abar[...] = cmd; — 已删除) */
|
||
|
||
g_abar[(base + PxIS) / 4] = 0xffffffffu;
|
||
g_abar[HBA_IS / 4] = (1u << port);
|
||
g_abar[(base + PxCI) / 4] = 1;
|
||
/* P0 修复: 写入超时改为 TSC deadline，与读取路径一致。
|
||
* 固定循环在实机 HDD 上会因 CPU 频率差异导致提前超时或死循环。 */
|
||
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 5000;
|
||
while (dkm_rdtsc() < deadline) {
|
||
if ((g_abar[(base + PxCI) / 4] & 1u) == 0) {
|
||
/* BUG-P1-1 fix: CI 清除后检查 TFD 错误位。
|
||
* TFD 低字节是 STS，正常值 0x50 (DRDY|DRQ) 非零但不是错误。
|
||
* 只有 STS bit0 (ERR) 或 ERR 寄存器 (bits15:8) 非零才是真正错误。
|
||
* BUG-20260730-P14: 原 `tfd & 0xFF` 误判 STS=0x50 为错误，导致写入
|
||
* 实际成功却被报告为失败，FAT32 写路径提前返回错误码。 */
|
||
u32 tfd = g_abar[(base + PxTFD) / 4];
|
||
u32 is = g_abar[(base + PxIS) / 4];
|
||
u8 sts = (u8)(tfd & 0xFF);
|
||
u8 err_reg = (u8)((tfd >> 8) & 0xFF);
|
||
if ((sts & 0x01) || err_reg) {
|
||
g_log->warn("[ahci] write error: TFD ERR");
|
||
log_hex("[ahci] TFD=", tfd);
|
||
log_hex("[ahci] PxIS=", is);
|
||
return -4;
|
||
}
|
||
return 0;
|
||
}
|
||
__asm__ volatile("pause");
|
||
}
|
||
|
||
g_log->warn("[ahci] WRITE timeout");
|
||
log_hex("[ahci] WRITE lba=", lba);
|
||
log_hex("[ahci] WRITE count=", count);
|
||
log_hex("[ahci] WRITE CI=", g_abar[(base + PxCI) / 4]);
|
||
log_hex("[ahci] WRITE IS=", g_abar[(base + PxIS) / 4]);
|
||
log_hex("[ahci] WRITE TFD=", g_abar[(base + PxTFD) / 4]);
|
||
return -3;
|
||
}
|
||
|
||
static int ahci_block_read(void *ctx, u64 lba, u32 count, void *buffer) {
|
||
(void)ctx;
|
||
if (g_ready_port == 0xffffffffu || !g_cmd_header) return -1;
|
||
DKM_STAT_INC(ahci_block_reads);
|
||
log_hex("[ahci] block_read lba=", lba);
|
||
log_hex("[ahci] block_read count=", count);
|
||
while (count > 0) {
|
||
u32 chunk = count > 8 ? 8 : count;
|
||
int status = ahci_read_blocks_internal(g_ready_port, g_cmd_header, &g_cmd_table, &g_data, lba, chunk, buffer);
|
||
if (status != 0) return status;
|
||
lba += chunk;
|
||
count -= chunk;
|
||
buffer = (void *)((u8 *)buffer + (chunk * 512));
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
static int ahci_block_write(void *ctx, u64 lba, u32 count, const void *buffer) {
|
||
(void)ctx;
|
||
if (g_ready_port == 0xffffffffu || !g_cmd_header) return -1;
|
||
DKM_STAT_INC(ahci_block_writes);
|
||
/* ensure port is in ST+FRE state */
|
||
u32 base = HBA_PORT_BASE + g_ready_port * HBA_PORT_SIZE;
|
||
u32 cmd = g_abar[(base + PxCMD) / 4];
|
||
if (!(cmd & AHCI_CMD_ST) || !(cmd & AHCI_CMD_FRE)) {
|
||
g_log->warn("[ahci] port not ready; restarting");
|
||
ahci_stop_port(g_ready_port);
|
||
ahci_start_port(g_ready_port);
|
||
}
|
||
while (count > 0) {
|
||
u32 chunk = count > 8 ? 8 : count;
|
||
int status = ahci_write_blocks_internal(g_ready_port, g_cmd_header, &g_cmd_table, &g_data, lba, chunk, buffer);
|
||
if (status != 0) return status;
|
||
lba += chunk;
|
||
count -= chunk;
|
||
buffer = (const void *)((const u8 *)buffer + (chunk * 512));
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
static int ahci_read_lba0(u32 port, ahci_cmd_header *hdr, struct dkm_dma_buffer *table, struct dkm_dma_buffer *data) {
|
||
int status = ahci_read_blocks_internal(port, hdr, table, data, 0, 1, data->virt);
|
||
if (status == 0) {
|
||
log_hex("[ahci] READ LBA0 port=", port);
|
||
log_hex("[ahci] READ LBA0 bytes=", hdr[0].prdbc);
|
||
log_hex("[ahci] READ LBA0 word0=", ((u16 *)data->virt)[0]);
|
||
log_hex("[ahci] READ LBA0 word255=", ((u16 *)data->virt)[255]);
|
||
return 1;
|
||
}
|
||
return status;
|
||
}
|
||
|
||
static int ahci_identify_port(u32 port, const struct dkm_dma_api *dma) {
|
||
if (!dma || !dma->alloc_pages) return -1;
|
||
|
||
u32 base = HBA_PORT_BASE + port * HBA_PORT_SIZE;
|
||
u32 ssts = g_abar[(base + PxSSTS) / 4];
|
||
u32 sig = g_abar[(base + PxSIG) / 4];
|
||
u32 det = ssts & 0xf;
|
||
u32 ipm = (ssts >> 8) & 0xf;
|
||
if (det != 3 || ipm != 1) return 0;
|
||
if (sig != 0x00000101u) {
|
||
g_log->info("[ahci] IDENTIFY skipped non-SATA port");
|
||
log_hex("[ahci] IDENTIFY skip port=", port);
|
||
log_hex("[ahci] IDENTIFY skip sig=", sig);
|
||
return 0;
|
||
}
|
||
|
||
struct dkm_dma_buffer clb;
|
||
struct dkm_dma_buffer fis;
|
||
struct dkm_dma_buffer table;
|
||
struct dkm_dma_buffer data;
|
||
if (dma->alloc_pages(1, 1024, 0x100000000ULL, &clb) != 0) return -2;
|
||
if (dma->alloc_pages(1, 256, 0x100000000ULL, &fis) != 0) return -3;
|
||
/* BUG-P0-1: sizeof(ahci_cmd_table)=144, 128B 缓冲区越界 16 字节。
|
||
* 改为 256B 满足对齐和大小要求，防止 ahci_zero 越界写。 */
|
||
if (dma->alloc_pages(1, 256, 0x100000000ULL, &table) != 0) return -4;
|
||
/* BUG-20260730-022: DMA data buffer must hold at least 8 sectors (4096B)
|
||
* for ahci_block_write chunked DMA. 512B caused 3584B out-of-bounds
|
||
* overwrite of adjacent kernel data on real hardware. */
|
||
if (dma->alloc_pages(1, 8192, 0x100000000ULL, &data) != 0) return -5;
|
||
|
||
/* P0 验证: DMA 缓冲区物理地址必须在 4GB 以下。
|
||
* AHCI 控制器 8086:2922 只支持 32 位 DMA，
|
||
* PRDT 地址超过 4GB 会导致 DMA 写入截断到错误物理地址。
|
||
* dma->alloc_pages 的上限参数 0x100000000ULL 已保证此约束，
|
||
* 但添加显式校验以防御 alloc_pages 实现变更。 */
|
||
if (clb.phys >= 0x100000000ULL || fis.phys >= 0x100000000ULL ||
|
||
table.phys >= 0x100000000ULL || data.phys >= 0x100000000ULL) {
|
||
g_log->warn("[ahci] DMA buffer above 4GB — 32-bit DMA will fail on this controller!");
|
||
log_hex("[ahci] clb.phys=", clb.phys);
|
||
log_hex("[ahci] fis.phys=", fis.phys);
|
||
log_hex("[ahci] table.phys=", table.phys);
|
||
log_hex("[ahci] data.phys=", data.phys);
|
||
}
|
||
|
||
if (ahci_stop_port(port) != 0) {
|
||
g_log->warn("[ahci] port stop timeout");
|
||
return -6;
|
||
}
|
||
|
||
g_abar[(base + PxCLB) / 4] = (u32)(clb.phys & 0xffffffffu);
|
||
g_abar[(base + PxCLB + 4) / 4] = (u32)(clb.phys >> 32);
|
||
g_abar[(base + PxFB) / 4] = (u32)(fis.phys & 0xffffffffu);
|
||
g_abar[(base + PxFB + 4) / 4] = (u32)(fis.phys >> 32);
|
||
|
||
ahci_cmd_header *hdr = (ahci_cmd_header *)clb.virt;
|
||
ahci_cmd_table *tbl = (ahci_cmd_table *)table.virt;
|
||
hdr[0].flags = 5; /* FIS length in DWORDS */
|
||
hdr[0].prdtl = 1;
|
||
hdr[0].ctba = (u32)(table.phys & 0xffffffffu);
|
||
hdr[0].ctbau = (u32)(table.phys >> 32);
|
||
tbl->prdt[0].dba = (u32)(data.phys & 0xffffffffu);
|
||
tbl->prdt[0].dbau = (u32)(data.phys >> 32);
|
||
tbl->prdt[0].dbc_i = (512 - 1) | (1u << 31);
|
||
|
||
u8 cmd = ATA_CMD_IDENTIFY;
|
||
ahci_fill_identify_fis(tbl->cfis, cmd);
|
||
|
||
g_abar[(base + PxIS) / 4] = 0xffffffffu;
|
||
g_abar[HBA_IS / 4] = (1u << port);
|
||
ahci_start_port(port);
|
||
|
||
g_abar[(base + PxCI) / 4] = 1;
|
||
/* BUG-P0-3: 固定 500 万次循环在 3GHz CPU 上仅 ~1-2ms，
|
||
* 不够 HDD 寻道。改为 TSC deadline 5s 超时，与 read/write 路径一致。 */
|
||
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 5000;
|
||
while (dkm_rdtsc() < deadline) {
|
||
if ((g_abar[(base + PxCI) / 4] & 1u) == 0) {
|
||
log_hex("[ahci] IDENTIFY port=", port);
|
||
log_hex("[ahci] IDENTIFY status=", g_abar[(base + PxTFD) / 4]);
|
||
log_hex("[ahci] IDENTIFY bytes=", hdr[0].prdbc);
|
||
u16 *id = (u16 *)data.virt;
|
||
log_hex("[ahci] IDENTIFY word0=", id[0]);
|
||
log_hex("[ahci] IDENTIFY word49=", id[49]);
|
||
log_hex("[ahci] IDENTIFY word83=", id[83]);
|
||
/* 容量必须在 ahci_read_lba0 复用 data 缓冲区前提取，
|
||
* 否则 IDENTIFY 数据被 LBA0 内容覆盖后容量丢失。 */
|
||
if ((id[83] & 0x0400u) && (id[103] | id[102] | id[101] | id[100])) {
|
||
/* LBA48: word100-103 小端 u16 序列拼接 */
|
||
g_sector_count = (u64)id[100] | ((u64)id[101] << 16)
|
||
| ((u64)id[102] << 32) | ((u64)id[103] << 48);
|
||
} else {
|
||
/* LBA28: word60-61 */
|
||
g_sector_count = (u64)id[60] | ((u64)id[61] << 16);
|
||
}
|
||
log_hex("[ahci] IDENTIFY sector_count=", g_sector_count);
|
||
ahci_read_lba0(port, hdr, &table, &data);
|
||
g_ready_port = port;
|
||
g_cmd_header = hdr;
|
||
g_cmd_table = table;
|
||
g_data = data;
|
||
return 1;
|
||
}
|
||
__asm__ volatile("pause");
|
||
}
|
||
|
||
g_log->warn("[ahci] IDENTIFY timeout");
|
||
log_hex("[ahci] IDENTIFY CI=", g_abar[(base + PxCI) / 4]);
|
||
log_hex("[ahci] IDENTIFY IS=", g_abar[(base + PxIS) / 4]);
|
||
log_hex("[ahci] IDENTIFY TFD=", g_abar[(base + PxTFD) / 4]);
|
||
return -7;
|
||
}
|
||
|
||
__attribute__((visibility("default")))
|
||
int driver_init(const struct dkm_kernel_api *api,
|
||
struct dkm_driver_handle *handle) {
|
||
(void)handle;
|
||
/* 实机要求: 先校准 TSC, 再使用基于 CPU 频率的延迟 */
|
||
dkm_tsc_calibrate();
|
||
|
||
if (!api || !api->log) return -1;
|
||
g_log = api->log;
|
||
dkm_instr_init(api);
|
||
|
||
g_log->info("[ahci] init begin");
|
||
|
||
u8 bus = 0;
|
||
u8 dev = 0;
|
||
u8 func = 0;
|
||
if (ahci_find(&bus, &dev, &func) != 0) {
|
||
g_log->warn("[ahci] AHCI controller not found");
|
||
g_log->info("[ahci] driver ready");
|
||
return 0;
|
||
}
|
||
|
||
g_log->info("[ahci] AHCI controller found");
|
||
log_hex("[ahci] bus=", bus);
|
||
log_hex("[ahci] dev=", dev);
|
||
log_hex("[ahci] func=", func);
|
||
|
||
u32 vd = pci_read(bus, dev, func, PCI_VENDOR_ID);
|
||
log_hex("[ahci] vendor=", vd & 0xffff);
|
||
log_hex("[ahci] device=", vd >> 16);
|
||
|
||
u32 command = pci_read(bus, dev, func, PCI_COMMAND) & 0xffff;
|
||
command |= PCI_CMD_MEM | PCI_CMD_BUSM;
|
||
pci_write(bus, dev, func, PCI_COMMAND, command);
|
||
log_hex("[ahci] PCI command=", command);
|
||
|
||
/* 读取 ABAR: BAR5 可能是 64-bit MMIO BAR，高位在 BAR4 */
|
||
u32 bar5_lo = pci_read(bus, dev, func, PCI_BAR5);
|
||
u64 abar_phys;
|
||
if ((bar5_lo & 0x7) == 0x4) {
|
||
/* 64-bit MMIO BAR: 高 32 位在 BAR4 */
|
||
u32 bar5_hi = pci_read(bus, dev, func, PCI_BAR4);
|
||
abar_phys = ((u64)bar5_hi << 32) | (bar5_lo & 0xFFFFFFF0u);
|
||
g_log->info("[ahci] ABAR is 64-bit MMIO BAR");
|
||
} else {
|
||
abar_phys = bar5_lo & 0xFFFFFFF0u;
|
||
}
|
||
u32 irq_line = pci_read(bus, dev, func, PCI_IRQ_LINE) & 0xff;
|
||
log_hex("[ahci] ABAR phys=", abar_phys);
|
||
log_hex("[ahci] PCI IRQ line=", irq_line);
|
||
|
||
if (!api->hhdm_offset || !abar_phys) {
|
||
g_log->warn("[ahci] missing HHDM or ABAR; skip MMIO enumeration");
|
||
g_log->info("[ahci] driver ready");
|
||
return 0;
|
||
}
|
||
|
||
g_abar = (volatile u32 *)(uintptr_t)(api->hhdm_offset + abar_phys);
|
||
DKM_TS_BEGIN(ts_abar);
|
||
log_hex("[ahci] HHDM offset=", api->hhdm_offset);
|
||
|
||
u32 cap = g_abar[HBA_CAP / 4];
|
||
u32 ghc = g_abar[HBA_GHC / 4];
|
||
u32 is = g_abar[HBA_IS / 4];
|
||
u32 pi = g_abar[HBA_PI / 4];
|
||
u32 vs = g_abar[HBA_VS / 4];
|
||
u32 cap2 = g_abar[HBA_CAP2 / 4];
|
||
u32 bohc = g_abar[HBA_BOHC / 4];
|
||
|
||
log_hex("[ahci] CAP=", cap);
|
||
log_hex("[ahci] GHC=", ghc);
|
||
log_hex("[ahci] IS=", is);
|
||
log_hex("[ahci] PI=", pi);
|
||
log_hex("[ahci] VS=", vs);
|
||
log_hex("[ahci] CAP2=", cap2);
|
||
log_hex("[ahci] BOHC=", bohc);
|
||
|
||
/* AHCI 1.3+ BOHC handoff: 如果 BIOS 仍拥有控制器, 请求所有权 */
|
||
if (bohc & 0x01) {
|
||
g_log->info("[ahci] BIOS owns HBA, requesting handoff");
|
||
g_abar[HBA_BOHC / 4] = bohc | 0x02; /* set OOS */
|
||
u64 deadline = dkm_rdtsc() + dkm_tsc_per_ms * 2000;
|
||
while (dkm_rdtsc() < deadline) {
|
||
if (!(g_abar[HBA_BOHC / 4] & 0x01)) break;
|
||
__asm__ volatile("pause");
|
||
}
|
||
if (g_abar[HBA_BOHC / 4] & 0x01) {
|
||
g_log->warn("[ahci] BIOS handoff timeout, proceeding anyway");
|
||
} else {
|
||
g_log->info("[ahci] BIOS handoff complete");
|
||
}
|
||
}
|
||
|
||
log_hex("[ahci] ports count=", (cap & 0x1f) + 1);
|
||
log_hex("[ahci] cmd slots=", ((cap >> 8) & 0x1f) + 1);
|
||
DKM_TS_END(ts_abar, g_log->info, "ahci_abar_mmio");
|
||
|
||
u32 implemented = 0;
|
||
for (u32 port = 0; port < 32; port++) {
|
||
if (pi & (1u << port)) {
|
||
implemented++;
|
||
DKM_STAT_INC(ahci_port_found);
|
||
ahci_log_port(port);
|
||
}
|
||
}
|
||
log_hex("[ahci] implemented ports=", implemented);
|
||
|
||
u32 identified = 0;
|
||
if (api->dma) {
|
||
for (u32 port = 0; port < 32; port++) {
|
||
if (pi & (1u << port)) {
|
||
int ident = ahci_identify_port(port, api->dma);
|
||
if (ident > 0) {
|
||
identified++;
|
||
DKM_STAT_INC(ahci_identify_ok);
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
g_log->warn("[ahci] DMA API unavailable; skip IDENTIFY");
|
||
}
|
||
log_hex("[ahci] identified ports=", identified);
|
||
if (identified && api->block && api->block->register_device) {
|
||
struct dkm_block_device_desc desc;
|
||
desc.name = "ahci0";
|
||
desc.sector_size = 512;
|
||
desc.sector_count = g_sector_count;
|
||
desc.ctx = 0;
|
||
desc.read = ahci_block_read;
|
||
desc.write = ahci_block_write;
|
||
int index = api->block->register_device(&desc);
|
||
log_hex("[ahci] block provider index=", (u64)(i64)index);
|
||
} else if (identified) {
|
||
g_log->warn("[ahci] block API unavailable; provider not registered");
|
||
}
|
||
|
||
g_log->info("[ahci] driver ready");
|
||
return 0;
|
||
}
|
||
|
||
__attribute__((visibility("default")))
|
||
int driver_exit(struct dkm_driver_handle *handle) {
|
||
(void)handle;
|
||
return 0;
|
||
}
|