From e599e9c20d35cb5f7653379ff73e4cda0b0a8070 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 7 Jul 2026 11:14:53 +0800 Subject: [PATCH 1/2] fix(resolver): don't force full GPU offload when model exceeds discrete VRAM llama.cpp's default n_gpu_layers=999 ("offload every layer") assumes the model fits in VRAM. On a discrete GPU with less VRAM than the model needs (e.g. qwen3.5-9b on a 4GB RTX 2050) it forces an impossible allocation and CUDA-OOMs at load, surfacing to users as a deployment that never becomes ready. Cap the engine-declared offload knob (offload_config_key) to CPU when a discrete GPU can't hold the model's footprint, and warn with the numbers. Unified-memory hosts (APU/GB10/Apple) share system RAM with the GPU, so full offload is correct there and is left untouched. The knob is read from catalog YAML, so no engine name or config key is hardcoded (INV-1/2). Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/knowledge/offload_test.go | 104 +++++++++++++++++++++++++++++ internal/knowledge/resolver.go | 50 ++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 internal/knowledge/offload_test.go diff --git a/internal/knowledge/offload_test.go b/internal/knowledge/offload_test.go new file mode 100644 index 00000000..977b17dd --- /dev/null +++ b/internal/knowledge/offload_test.go @@ -0,0 +1,104 @@ +package knowledge + +import "testing" + +// baseOffloadInputs builds a resolved/engine/variant/hw set representing a +// llamacpp GGUF variant that declares "fits any GPU" (vram_min_mib=0) with a +// full-GPU-offload default (n_gpu_layers=999) and a real RAM footprint. +func baseOffloadInputs() (*ResolvedConfig, *EngineAsset, *ModelVariant) { + resolved := &ResolvedConfig{ + Config: map[string]any{"n_gpu_layers": 999}, + Provenance: map[string]string{"n_gpu_layers": "L0"}, + } + engine := &EngineAsset{Amplifier: EngineAmplifier{OffloadConfigKey: "n_gpu_layers"}} + variant := &ModelVariant{Hardware: ModelVariantHardware{RAMMinMiB: 7168}} + return resolved, engine, variant +} + +func TestCapGPUOffloadDowngradesOnSmallDiscreteGPU(t *testing.T) { + resolved, engine, variant := baseOffloadInputs() + hw := HardwareInfo{GPUVRAMMiB: 4096, UnifiedMemory: false} + + capGPUOffloadForVRAM(resolved, engine, variant, hw) + + if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 0 { + t.Errorf("n_gpu_layers = %v, want 0 (CPU) when model (7168 MiB) exceeds GPU VRAM (4096 MiB)", got) + } + if !resolved.OffloadPath { + t.Error("OffloadPath = false, want true after CPU downgrade") + } + if len(resolved.Warnings) != 1 { + t.Errorf("Warnings = %d, want 1 explaining the CPU fallback", len(resolved.Warnings)) + } +} + +func TestCapGPUOffloadLeavesUnifiedMemoryAlone(t *testing.T) { + resolved, engine, variant := baseOffloadInputs() + // Unified-memory APU/GB10/Apple: the GPU shares system RAM, so full offload + // is correct even when the footprint exceeds nominal VRAM. + hw := HardwareInfo{GPUVRAMMiB: 4096, UnifiedMemory: true} + + capGPUOffloadForVRAM(resolved, engine, variant, hw) + + if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 999 { + t.Errorf("n_gpu_layers = %v, want 999 (untouched) on unified memory", got) + } + if len(resolved.Warnings) != 0 { + t.Errorf("Warnings = %d, want 0 on unified memory", len(resolved.Warnings)) + } +} + +func TestCapGPUOffloadLeavesFittingGPUAlone(t *testing.T) { + resolved, engine, variant := baseOffloadInputs() + hw := HardwareInfo{GPUVRAMMiB: 49152, UnifiedMemory: false} // 48 GB card fits 7 GB model + + capGPUOffloadForVRAM(resolved, engine, variant, hw) + + if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 999 { + t.Errorf("n_gpu_layers = %v, want 999 (untouched) when model fits VRAM", got) + } +} + +func TestCapGPUOffloadRespectsUserOverride(t *testing.T) { + resolved, engine, variant := baseOffloadInputs() + resolved.Config["n_gpu_layers"] = 50 + resolved.Provenance["n_gpu_layers"] = "L1" // user set it explicitly + hw := HardwareInfo{GPUVRAMMiB: 4096, UnifiedMemory: false} + + capGPUOffloadForVRAM(resolved, engine, variant, hw) + + if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 50 { + t.Errorf("n_gpu_layers = %v, want 50 (user override respected)", got) + } +} + +func TestCapGPUOffloadNoopWithoutOffloadKey(t *testing.T) { + resolved, engine, variant := baseOffloadInputs() + engine.Amplifier.OffloadConfigKey = "" // engine declares no offload knob + hw := HardwareInfo{GPUVRAMMiB: 4096, UnifiedMemory: false} + + capGPUOffloadForVRAM(resolved, engine, variant, hw) + + if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 999 { + t.Errorf("n_gpu_layers = %v, want 999 (untouched) when no offload_config_key", got) + } +} + +func TestCheckFitSurfacesResolvedWarnings(t *testing.T) { + resolved := &ResolvedConfig{ + Config: map[string]any{}, + Warnings: []string{"model needs ~7168 MiB but this GPU has 4096 MiB VRAM"}, + } + + fit := CheckFit(resolved, HardwareInfo{}) + + found := false + for _, w := range fit.Warnings { + if w == resolved.Warnings[0] { + found = true + } + } + if !found { + t.Errorf("CheckFit did not surface resolve-time warning; got %v", fit.Warnings) + } +} diff --git a/internal/knowledge/resolver.go b/internal/knowledge/resolver.go index 906a61c0..49067f96 100644 --- a/internal/knowledge/resolver.go +++ b/internal/knowledge/resolver.go @@ -97,6 +97,10 @@ type ResolvedConfig struct { // Performance reference (K4 — historical data or YAML estimate) PerformanceRef *PerformanceReference + + // Warnings collected during resolution (e.g. GPU-offload downgrade). Surfaced + // to the user via CheckFit's FitReport so resolve-time decisions are visible. + Warnings []string } // PerformanceReference attaches known performance data to a resolved config. @@ -295,6 +299,7 @@ func (c *Catalog) Resolve(hw HardwareInfo, modelName, engineType string, userOve resolved.EstimatedVRAMMiB = variant.Hardware.VRAMMinMiB } resolved.ResourceEstimate = estimateResources(engine, variant, hw) + capGPUOffloadForVRAM(resolved, engine, variant, hw) } // Container memory guardrail. On unified-memory hosts (GB10, Apple M-series, @@ -1596,6 +1601,47 @@ func (c *Catalog) UpsertSyntheticModel(ma ModelAsset) { c.ModelAssets = append(c.ModelAssets, ma) } +// capGPUOffloadForVRAM downgrades an engine's GPU-offload knob to CPU when a +// discrete GPU cannot hold the model. llama.cpp's default n_gpu_layers=999 +// ("offload every layer") assumes the model fits VRAM; on a discrete GPU with +// less VRAM than the model needs it forces an impossible allocation → CUDA OOM +// at load, which surfaces to users as a deployment that never becomes ready. +// Unified-memory hosts (APU/GB10/Apple) share system RAM with the GPU, so full +// offload is correct there and left untouched. The knob is identified by the +// engine-declared offload_config_key, so this stays engine-agnostic (INV-1/2): +// no engine name or config key is hardcoded here. +func capGPUOffloadForVRAM(resolved *ResolvedConfig, engine *EngineAsset, variant *ModelVariant, hw HardwareInfo) { + if resolved == nil || engine == nil || variant == nil { + return + } + key := engine.Amplifier.OffloadConfigKey + if key == "" || hw.UnifiedMemory || hw.GPUVRAMMiB <= 0 { + return + } + // Model memory footprint. RAMMinMiB is the catalog's footprint proxy, used + // precisely when vram_min_mib=0 (a GGUF variant declared to "fit any GPU" + // via CPU offload); fall back to any explicit VRAM estimate. + footprint := variant.Hardware.RAMMinMiB + if resolved.EstimatedVRAMMiB > footprint { + footprint = resolved.EstimatedVRAMMiB + } + if footprint <= 0 || footprint <= hw.GPUVRAMMiB { + return // fits, or footprint unknown — full GPU offload is correct + } + if resolved.Provenance[key] == "L1" { + return // user set the offload knob explicitly — respect it + } + if cur, ok := resolved.Config[key]; ok && toFloat64(cur) == 0 { + return // already CPU-only + } + resolved.Config[key] = 0 + resolved.Provenance[key] = "L0-fit" + resolved.OffloadPath = true + resolved.Warnings = append(resolved.Warnings, fmt.Sprintf( + "model needs ~%d MiB but this GPU has %d MiB VRAM; disabling GPU offload and running on CPU (much slower). Use a smaller model or a larger-VRAM GPU for GPU acceleration.", + footprint, hw.GPUVRAMMiB)) +} + // estimateResources computes cost(path, R) — the full resource consumption estimate. func estimateResources(engine *EngineAsset, variant *ModelVariant, hw HardwareInfo) *ResourceEstimate { perf := variant.ParsedExpectedPerf() @@ -1644,6 +1690,10 @@ var gmuKeys = []string{"gpu_memory_utilization", "mem_fraction_static"} func CheckFit(resolved *ResolvedConfig, hw HardwareInfo) *FitReport { r := &FitReport{Fit: true, Adjustments: make(map[string]any)} + // Surface warnings collected during resolution (e.g. GPU-offload downgrade) + // even if a hard-fail check below returns early. + r.Warnings = append(r.Warnings, resolved.Warnings...) + // Unified memory guard: GPU allocation directly reduces available system memory. // Enforce minimum OS reserve to prevent starvation / swap thrashing. if hw.UnifiedMemory && hw.RAMTotalMiB > 0 { From bd56ca7be3c477acbd9ce118ac8c95cc29cb1196 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 7 Jul 2026 11:14:53 +0800 Subject: [PATCH 2/2] fix(run): pre-flight disk space before model download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A model pull streamed straight to disk with no free-space check. On a small disk a multi-GB GGUF could fill the filesystem past k3s's disk-pressure eviction threshold, tainting the node NoSchedule so the pod stayed Pending forever — surfacing only as "Timed out waiting for deployment to be ready". Guard the download: the moment its total size is known (reported by the existing downloader progress, before transfer for HF/ModelScope), abort if it won't fit while preserving a disk-pressure reserve (10% of the volume, 2 GiB floor), returning a clear, fatal, pre-transfer error instead of a disk-filling hang. Adds hal.DiskUsage for cross-platform free/total bytes. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/aima/diskcheck.go | 28 ++++++++++++++++++ cmd/aima/diskcheck_test.go | 59 ++++++++++++++++++++++++++++++++++++++ cmd/aima/main.go | 30 ++++++++++++++++++- internal/hal/detect.go | 6 ++++ 4 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 cmd/aima/diskcheck.go create mode 100644 cmd/aima/diskcheck_test.go diff --git a/cmd/aima/diskcheck.go b/cmd/aima/diskcheck.go new file mode 100644 index 00000000..74bb13ad --- /dev/null +++ b/cmd/aima/diskcheck.go @@ -0,0 +1,28 @@ +package main + +// diskReserveMiB is the free space kept beyond a model download so the node +// does not cross k3s's disk-pressure eviction threshold (kubelet defaults evict +// at nodefs<10% / imagefs<15%, which taints the node NoSchedule and leaves pods +// stuck Pending). Reserve 10% of the volume, with a 2 GiB floor for small disks. +func diskReserveMiB(totalMiB int64) int64 { + reserve := totalMiB / 10 + if reserve < 2048 { + reserve = 2048 + } + return reserve +} + +// enoughDiskForDownload reports whether a download of requiredMiB fits on a +// filesystem with freeMiB free / totalMiB total while preserving the reserve. +// requiredMiB<=0 means the size is unknown, in which case it cannot gate and +// returns true (best-effort). Returns the shortfall in MiB when it does not fit. +func enoughDiskForDownload(requiredMiB, freeMiB, totalMiB int64) (ok bool, shortfallMiB int64) { + if requiredMiB <= 0 { + return true, 0 + } + need := requiredMiB + diskReserveMiB(totalMiB) + if freeMiB >= need { + return true, 0 + } + return false, need - freeMiB +} diff --git a/cmd/aima/diskcheck_test.go b/cmd/aima/diskcheck_test.go new file mode 100644 index 00000000..805f5329 --- /dev/null +++ b/cmd/aima/diskcheck_test.go @@ -0,0 +1,59 @@ +package main + +import "testing" + +func TestEnoughDiskForDownload(t *testing.T) { + tests := []struct { + name string + requiredMiB int64 + freeMiB int64 + totalMiB int64 + wantOK bool + wantShortfallMiB int64 + }{ + { + name: "plenty of space", + requiredMiB: 5000, freeMiB: 200000, totalMiB: 500000, + wantOK: true, + }, + { + // 5 GB download, only 6 GB free on a 100 GB disk: reserve is 10% + // (10 GB), so it needs 15 GB but has 6 GB — would trip disk-pressure. + name: "download would trip disk-pressure", + requiredMiB: 5000, freeMiB: 6000, totalMiB: 100000, + wantOK: false, wantShortfallMiB: 9000, + }, + { + // Small disk: reserve floors at 2048 MiB rather than 10%. + name: "reserve floor on small disk", + requiredMiB: 1000, freeMiB: 2500, totalMiB: 10000, + wantOK: false, wantShortfallMiB: 548, + }, + { + // Unknown download size: cannot gate, must not false-block. + name: "unknown size does not gate", + requiredMiB: 0, freeMiB: 100, totalMiB: 1000, + wantOK: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ok, shortfall := enoughDiskForDownload(tt.requiredMiB, tt.freeMiB, tt.totalMiB) + if ok != tt.wantOK { + t.Errorf("ok = %v, want %v", ok, tt.wantOK) + } + if !ok && shortfall != tt.wantShortfallMiB { + t.Errorf("shortfall = %d, want %d", shortfall, tt.wantShortfallMiB) + } + }) + } +} + +func TestDiskReserveMiB(t *testing.T) { + if got := diskReserveMiB(500000); got != 50000 { + t.Errorf("diskReserveMiB(500000) = %d, want 50000 (10%%)", got) + } + if got := diskReserveMiB(10000); got != 2048 { + t.Errorf("diskReserveMiB(10000) = %d, want 2048 (floor)", got) + } +} diff --git a/cmd/aima/main.go b/cmd/aima/main.go index bbd1445c..6243171e 100644 --- a/cmd/aima/main.go +++ b/cmd/aima/main.go @@ -1517,6 +1517,10 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps { // Step 3: Pull model (non-fatal — may be local or pre-installed). Use // pullModelCore directly so byte-level progress can flow back to the // caller via onModelProgress (the deps.PullModel closure swallows it). + // A disk guard aborts the moment the download's total size is known to + // not fit (keeping a disk-pressure reserve), turning a disk-filling hang + // — which taints the k3s node NoSchedule so the pod never starts — into + // a clear, fatal, pre-transfer error. if !noPull { notify("pulling_model", model) modelStatus := func(phase, msg string) { @@ -1524,7 +1528,31 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps { notify("pulling_model", msg) } } - if err := pullModelCore(ctx, model, modelStatus, onModelProgress); err != nil { + freeMiB, totalMiB := hal.DiskUsage(dataDir) + pullCtx, cancelPull := context.WithCancel(ctx) + var diskErr error + diskChecked := false + guardedProgress := func(downloaded, total int64) { + if !diskChecked && total > 0 && freeMiB > 0 { + diskChecked = true + requiredMiB := total / (1024 * 1024) + if ok, shortfall := enoughDiskForDownload(requiredMiB, freeMiB, totalMiB); !ok { + diskErr = fmt.Errorf("not enough disk space to download %s: needs ~%d MiB plus reserve but only %d MiB free on %s (short by ~%d MiB); free up space or choose a smaller model", + model, requiredMiB, freeMiB, dataDir, shortfall) + cancelPull() + return + } + } + if onModelProgress != nil { + onModelProgress(downloaded, total) + } + } + err := pullModelCore(pullCtx, model, modelStatus, guardedProgress) + cancelPull() + if diskErr != nil { + return nil, diskErr + } + if err != nil { notify("model_skip", err.Error()) } } diff --git a/internal/hal/detect.go b/internal/hal/detect.go index 12425c4d..35438348 100644 --- a/internal/hal/detect.go +++ b/internal/hal/detect.go @@ -142,6 +142,12 @@ func detectStorage() StorageInfo { } } +// DiskUsage returns free and total space (MiB) on the filesystem containing +// path, or (0, 0) if it cannot be determined. Cross-platform via diskStats. +func DiskUsage(path string) (freeMiB, totalMiB int64) { + return diskStats(path) +} + // collectMetricsWithRunner gathers real-time metrics using given CommandRunner. func collectMetricsWithRunner(ctx context.Context, runner CommandRunner) (*Metrics, error) { m := &Metrics{}