diff --git a/.gitignore b/.gitignore index 1543317..ea5eaaf 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,11 @@ gddr6 build/ *.o *.a + +# local reverse-engineering probes / scratch tools +probe_* +probe5090* +scan_therm* +measure_gentle* +memload* +!*.md diff --git a/README.md b/README.md index 042f088..df3921b 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,19 @@ Reads GDDR6/GDDR6X VRAM memory temperatures from multiple supported NVIDIA GPUs found in a host Linux system. These findings are based on reverse engineering of the NVIDIA GPU Linux driver. +### Experimental: RTX 5090 (GDDR7) + +Experimental support for the RTX 5090 (Blackwell / GB202, GDDR7) is included. It reads the per-module +DRAM sensors directly and reports the hotspot (hottest module) by default: + +``` +sudo gddr6 # VRAM hotspot temperature +sudo gddr6 --per-module # each of the 8 GDDR7 modules separately +``` + +This is reverse-engineered and unofficial (NVIDIA does not expose memory temperature via nvidia-smi/NVML +on this card) — treat the readings as approximate. + ## Prerequisites @@ -46,6 +59,7 @@ sudo gddr6 ``` ## Supported GPUs +- RTX 5090 (GB202) — GDDR7, experimental (see above) - RTX 4090 (AD102) - RTX 4080 Super (AD103) - RTX 4080 (AD103) diff --git a/app/src/app.c b/app/src/app.c index 2292301..35e1dc2 100644 --- a/app/src/app.c +++ b/app/src/app.c @@ -2,6 +2,7 @@ #include "gddr6.h" #include #include +#include #include void register_signal_handlers(void) @@ -18,6 +19,21 @@ void register_signal_handlers(void) int main(int argc, char **argv) { + int per_module = 0; + for (int i = 1; i < argc; i++) + { + if (strcmp(argv[i], "--per-module") == 0) + per_module = 1; + else if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) + { + printf("Usage: %s [--per-module]\n" + " (default) show VRAM temperature (Blackwell: hottest module)\n" + " --per-module show each GDDR7 module separately (Blackwell only)\n", + argv[0]); + return 0; + } + } + register_signal_handlers(); gddr6_init(); int num_devs = gddr6_detect_compatible_gpus(); @@ -29,7 +45,7 @@ int main(int argc, char **argv) } gddr6_memory_map(); - gddr6_monitor_temperatures(); + gddr6_monitor_temperatures(per_module); return 0; } diff --git a/lib/include/gddr6.h b/lib/include/gddr6.h index 957d71b..b4ca051 100644 --- a/lib/include/gddr6.h +++ b/lib/include/gddr6.h @@ -4,11 +4,19 @@ #include +// How to convert a raw register read into degrees Celsius. +enum temp_decode { + DECODE_ADA = 0, // Ada/Ampere: (raw & 0xfff) / 32 + DECODE_GDDR_MRCODE, // Blackwell FBPA DQR: byte in bits 23:16 is a GDDR temp + // MR-code; C = (code-20)*2 for code>19, else -(40-code*2) +}; + struct device { uint32_t bar0; uint8_t bus, dev, func; uint32_t offset; + enum temp_decode decode; // how to turn the raw reg into Celsius uint16_t dev_id; const char *vram; const char *arch; @@ -27,7 +35,7 @@ struct gddr6_ctx { void gddr6_init(void); void gddr6_memory_map(void); void gddr6_cleanup(int signal); -void gddr6_monitor_temperatures(void); +void gddr6_monitor_temperatures(int per_module); int gddr6_detect_compatible_gpus(void); #endif // GDDR6_H diff --git a/lib/src/gddr6.c b/lib/src/gddr6.c index 4f83e7b..95fef0f 100644 --- a/lib/src/gddr6.c +++ b/lib/src/gddr6.c @@ -11,7 +11,6 @@ #include #include #include -#include #define PG_SZ sysconf(_SC_PAGE_SIZE) #define PRINT_ERROR() \ @@ -23,6 +22,7 @@ #define MAX_DEVICES 32 struct gddr6_ctx ctx = {0}; +// Ada/Ampere GPUs: temperature field is bits [11:0], Celsius = field / 32. struct device dev_table[] = { { .offset = 0x0000E2A8, .dev_id = 0x2684, .vram = "GDDR6X", .arch = "AD102", .name = "RTX 4090" }, @@ -49,6 +49,14 @@ struct device dev_table[] = { .offset = 0x0000E2A8, .dev_id = 0x27b8, .vram = "GDDR6", .arch = "AD104", .name = "L4" }, { .offset = 0x0000E2A8, .dev_id = 0x26b9, .vram = "GDDR6", .arch = "AD102", .name = "L40S" }, { .offset = 0x0000E2A8, .dev_id = 0x2236, .vram = "GDDR6", .arch = "GA102", .name = "A10" }, + // Blackwell GDDR7 memory temperature: the raw FBPA DRAM sensor at + // NV_PFB_FBPA_DQR_STATUS_DQ_IC0_SUBP0 (0x9A24C0) - the register the FBFALCON + // firmware reads. NOT PLM-locked; reads valid data from userspace (validity + // bit 24 of 0x9A24D0). The value is a per-device GDDR temp MR-code in bits + // 23:16; DECODE_GDDR_MRCODE converts it to Celsius. (The documented mem-temp + // reg 0x9A44B0 is PLM-locked and its 0xE2A8 scratch mirror is unpopulated on + // this card, so we read the raw sensor directly.) + { .offset = 0x009A24C0, .decode = DECODE_GDDR_MRCODE, .dev_id = 0x2b85, .vram = "GDDR7", .arch = "GB202", .name = "RTX 5090" }, }; void gddr6_init(void) @@ -125,8 +133,77 @@ void gddr6_memory_map(void) } } -void gddr6_monitor_temperatures(void) +// Convert a raw register value to degrees Celsius per the device's decode. +static int decode_temp(enum temp_decode decode, uint32_t raw) { + switch (decode) + { + case DECODE_GDDR_MRCODE: + { + // GDDR temp MR-code in bits 23:16 (see NV_PFB_FBPA_DQR_STATUS_DQ). + // code 20 = 0 C, +2 C per unit above 20; below 20 is negative. + int code = (raw >> 16) & 0xFF; + if (code > 80) code = 80; + return (code > 19) ? (code - 20) * 2 : -(40 - code * 2); + } + case DECODE_ADA: + default: + return (raw & 0x00000fff) / 0x20; + } +} + +// Blackwell GDDR7 per-memory-partition (module) DQR sensors. Module p lives at +// BAR0 + DQR_MODULE0 + p*DQR_STRIDE; validity nibble (all 4 IC/subp valid = 0xF) +// is at +DQR_VLD_OFF. Unlike the single pre-mapped register, these span several +// pages, so they are read on demand with a fresh page-aligned mmap. +#define DQR_MODULE0 0x009024C0u +#define DQR_VLD_OFF (0x009024D0u - 0x009024C0u) // +0x10 +#define DQR_STRIDE 0x00004000u +#define DQR_MAX_MODULES 16 + +// Read one 32-bit MMIO register at BAR0+off via a fresh read-only page mmap. +// Returns 0 on success. Used only for the on-demand per-module GDDR7 reads. +static int read_bar0_reg(uint32_t bar0, uint32_t off, uint32_t *out) +{ + long pg = PG_SZ; + uint64_t phys = (uint64_t)bar0 + off; + uint64_t base = phys & ~((uint64_t)pg - 1); + volatile void *map = mmap(0, pg, PROT_READ, MAP_SHARED, ctx.fd, base); + if (map == MAP_FAILED) return -1; + *out = *(volatile uint32_t *)((const uint8_t *)map + (phys - base)); + munmap((void *)map, pg); + return 0; +} + +// Read the GDDR7 modules for a Blackwell device. Fills temps[]/present[] for up +// to DQR_MAX_MODULES, returns the module count found and the hottest temp in +// *hottest. A module counts as present only if all 4 DQR valid bits are set and +// the data word is not the 0xBADF.... poison sentinel. +static int gddr7_read_modules(uint32_t bar0, int temps[], int *hottest) +{ + int count = 0, hot = -128; + for (int p = 0; p < DQR_MAX_MODULES; p++) + { + uint32_t off = DQR_MODULE0 + (uint32_t)p * DQR_STRIDE; + uint32_t vld = 0, dq = 0; + if (read_bar0_reg(bar0, off + DQR_VLD_OFF, &vld) != 0) continue; + if (read_bar0_reg(bar0, off, &dq) != 0) continue; + + int all_valid = (((vld >> 24) & 0xF) == 0xF); + int poison = ((dq & 0xFFFF0000u) == 0xBADF0000u); + if (!all_valid || poison) continue; + + int c = decode_temp(DECODE_GDDR_MRCODE, dq); + temps[count++] = c; + if (c > hot) hot = c; + } + *hottest = hot; + return count; +} + +void gddr6_monitor_temperatures(int per_module) +{ + int temps[DQR_MAX_MODULES]; while (1) { printf("\rVRAM Temps: |"); for (uint32_t i = 0; i < ctx.num_devices; i++) @@ -136,11 +213,31 @@ void gddr6_monitor_temperatures(void) continue; } - void *virt_addr = (uint8_t *) ctx.devices[i].mapped_addr + (ctx.devices[i].phys_addr - ctx.devices[i].base_offset); - uint32_t read_result = *((uint32_t *)virt_addr); - uint32_t temp = ((read_result & 0x00000fff) / 0x20); + // Blackwell GDDR7: per-module DQR sensors. Default shows the hotspot + // (max across modules); --per-module lists each module. + if (ctx.devices[i].decode == DECODE_GDDR_MRCODE) + { + int hottest = 0; + int n = gddr7_read_modules(ctx.devices[i].bar0, temps, &hottest); + if (n == 0) { printf(" n/a |"); continue; } - printf(" %3u°C |", temp); + if (per_module) + { + for (int m = 0; m < n; m++) + printf(" m%d=%3d°C |", m, temps[m]); + } + else + { + printf(" %3d°C (hotspot) |", hottest); + } + continue; + } + + // Ada/Ampere: single pre-mapped VRAM register. + void *virt_addr = (uint8_t *) ctx.devices[i].mapped_addr + (ctx.devices[i].phys_addr - ctx.devices[i].base_offset); + uint32_t read_result = *((uint32_t *)virt_addr); + int temp = decode_temp(ctx.devices[i].decode, read_result); + printf(" %3d°C |", temp); } fflush(stdout); sleep(1);