fix: pre-flight disk space + cap GPU offload so small-GPU runs don't hang

cmd/aima/diskcheck.go

@@ -0,0 +1,28 @@
+package main
+
+// diskReserveMiB is the free space kept beyond a model download so the node
+// does not cross k3s's disk-pressure eviction threshold (kubelet defaults evict
+// at nodefs<10% / imagefs<15%, which taints the node NoSchedule and leaves pods
+// stuck Pending). Reserve 10% of the volume, with a 2 GiB floor for small disks.
+func diskReserveMiB(totalMiB int64) int64 {
+	reserve := totalMiB / 10
+	if reserve < 2048 {
+		reserve = 2048
+	}
+	return reserve
+}
+
+// enoughDiskForDownload reports whether a download of requiredMiB fits on a
+// filesystem with freeMiB free / totalMiB total while preserving the reserve.
+// requiredMiB<=0 means the size is unknown, in which case it cannot gate and
+// returns true (best-effort). Returns the shortfall in MiB when it does not fit.
+func enoughDiskForDownload(requiredMiB, freeMiB, totalMiB int64) (ok bool, shortfallMiB int64) {
+	if requiredMiB <= 0 {
+		return true, 0
+	}
+	need := requiredMiB + diskReserveMiB(totalMiB)
+	if freeMiB >= need {
+		return true, 0
+	}
+	return false, need - freeMiB
+}

cmd/aima/diskcheck_test.go

@@ -0,0 +1,59 @@
+package main
+
+import "testing"
+
+func TestEnoughDiskForDownload(t *testing.T) {
+	tests := []struct {
+		name             string
+		requiredMiB      int64
+		freeMiB          int64
+		totalMiB         int64
+		wantOK           bool
+		wantShortfallMiB int64
+	}{
+		{
+			name:        "plenty of space",
+			requiredMiB: 5000, freeMiB: 200000, totalMiB: 500000,
+			wantOK: true,
+		},
+		{
+			// 5 GB download, only 6 GB free on a 100 GB disk: reserve is 10%
+			// (10 GB), so it needs 15 GB but has 6 GB — would trip disk-pressure.
+			name:        "download would trip disk-pressure",
+			requiredMiB: 5000, freeMiB: 6000, totalMiB: 100000,
+			wantOK: false, wantShortfallMiB: 9000,
+		},
+		{
+			// Small disk: reserve floors at 2048 MiB rather than 10%.
+			name:        "reserve floor on small disk",
+			requiredMiB: 1000, freeMiB: 2500, totalMiB: 10000,
+			wantOK: false, wantShortfallMiB: 548,
+		},
+		{
+			// Unknown download size: cannot gate, must not false-block.
+			name:        "unknown size does not gate",
+			requiredMiB: 0, freeMiB: 100, totalMiB: 1000,
+			wantOK: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ok, shortfall := enoughDiskForDownload(tt.requiredMiB, tt.freeMiB, tt.totalMiB)
+			if ok != tt.wantOK {
+				t.Errorf("ok = %v, want %v", ok, tt.wantOK)
+			}
+			if !ok && shortfall != tt.wantShortfallMiB {
+				t.Errorf("shortfall = %d, want %d", shortfall, tt.wantShortfallMiB)
+			}
+		})
+	}
+}
+
+func TestDiskReserveMiB(t *testing.T) {
+	if got := diskReserveMiB(500000); got != 50000 {
+		t.Errorf("diskReserveMiB(500000) = %d, want 50000 (10%%)", got)
+	}
+	if got := diskReserveMiB(10000); got != 2048 {
+		t.Errorf("diskReserveMiB(10000) = %d, want 2048 (floor)", got)
+	}
+}

cmd/aima/main.go

@@ -1517,14 +1517,42 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps {
 		// Step 3: Pull model (non-fatal — may be local or pre-installed). Use
 		// pullModelCore directly so byte-level progress can flow back to the
 		// caller via onModelProgress (the deps.PullModel closure swallows it).
+		// A disk guard aborts the moment the download's total size is known to
+		// not fit (keeping a disk-pressure reserve), turning a disk-filling hang
+		// — which taints the k3s node NoSchedule so the pod never starts — into
+		// a clear, fatal, pre-transfer error.
 		if !noPull {
 			notify("pulling_model", model)
 			modelStatus := func(phase, msg string) {
 				if msg != "" {
 					notify("pulling_model", msg)
 				}
 			}
-			if err := pullModelCore(ctx, model, modelStatus, onModelProgress); err != nil {
+			freeMiB, totalMiB := hal.DiskUsage(dataDir)
+			pullCtx, cancelPull := context.WithCancel(ctx)
+			var diskErr error
+			diskChecked := false
+			guardedProgress := func(downloaded, total int64) {
+				if !diskChecked && total > 0 && freeMiB > 0 {
+					diskChecked = true
+					requiredMiB := total / (1024 * 1024)
+					if ok, shortfall := enoughDiskForDownload(requiredMiB, freeMiB, totalMiB); !ok {
+						diskErr = fmt.Errorf("not enough disk space to download %s: needs ~%d MiB plus reserve but only %d MiB free on %s (short by ~%d MiB); free up space or choose a smaller model",
+							model, requiredMiB, freeMiB, dataDir, shortfall)
+						cancelPull()
+						return
+					}
+				}
+				if onModelProgress != nil {
+					onModelProgress(downloaded, total)
+				}
+			}
+			err := pullModelCore(pullCtx, model, modelStatus, guardedProgress)
+			cancelPull()
+			if diskErr != nil {
+				return nil, diskErr
+			}
+			if err != nil {
 				notify("model_skip", err.Error())
 			}
 		}

internal/hal/detect.go

@@ -142,6 +142,12 @@ func detectStorage() StorageInfo {
 	}
 }
 
+// DiskUsage returns free and total space (MiB) on the filesystem containing
+// path, or (0, 0) if it cannot be determined. Cross-platform via diskStats.
+func DiskUsage(path string) (freeMiB, totalMiB int64) {
+	return diskStats(path)
+}
+
 // collectMetricsWithRunner gathers real-time metrics using given CommandRunner.
 func collectMetricsWithRunner(ctx context.Context, runner CommandRunner) (*Metrics, error) {
 	m := &Metrics{}

internal/knowledge/offload_test.go

@@ -0,0 +1,104 @@
+package knowledge
+
+import "testing"
+
+// baseOffloadInputs builds a resolved/engine/variant/hw set representing a
+// llamacpp GGUF variant that declares "fits any GPU" (vram_min_mib=0) with a
+// full-GPU-offload default (n_gpu_layers=999) and a real RAM footprint.
+func baseOffloadInputs() (*ResolvedConfig, *EngineAsset, *ModelVariant) {
+	resolved := &ResolvedConfig{
+		Config:     map[string]any{"n_gpu_layers": 999},
+		Provenance: map[string]string{"n_gpu_layers": "L0"},
+	}
+	engine := &EngineAsset{Amplifier: EngineAmplifier{OffloadConfigKey: "n_gpu_layers"}}
+	variant := &ModelVariant{Hardware: ModelVariantHardware{RAMMinMiB: 7168}}
+	return resolved, engine, variant
+}
+
+func TestCapGPUOffloadDowngradesOnSmallDiscreteGPU(t *testing.T) {
+	resolved, engine, variant := baseOffloadInputs()
+	hw := HardwareInfo{GPUVRAMMiB: 4096, UnifiedMemory: false}
+
+	capGPUOffloadForVRAM(resolved, engine, variant, hw)
+
+	if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 0 {
+		t.Errorf("n_gpu_layers = %v, want 0 (CPU) when model (7168 MiB) exceeds GPU VRAM (4096 MiB)", got)
+	}
+	if !resolved.OffloadPath {
+		t.Error("OffloadPath = false, want true after CPU downgrade")
+	}
+	if len(resolved.Warnings) != 1 {
+		t.Errorf("Warnings = %d, want 1 explaining the CPU fallback", len(resolved.Warnings))
+	}
+}
+
+func TestCapGPUOffloadLeavesUnifiedMemoryAlone(t *testing.T) {
+	resolved, engine, variant := baseOffloadInputs()
+	// Unified-memory APU/GB10/Apple: the GPU shares system RAM, so full offload
+	// is correct even when the footprint exceeds nominal VRAM.
+	hw := HardwareInfo{GPUVRAMMiB: 4096, UnifiedMemory: true}
+
+	capGPUOffloadForVRAM(resolved, engine, variant, hw)
+
+	if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 999 {
+		t.Errorf("n_gpu_layers = %v, want 999 (untouched) on unified memory", got)
+	}
+	if len(resolved.Warnings) != 0 {
+		t.Errorf("Warnings = %d, want 0 on unified memory", len(resolved.Warnings))
+	}
+}
+
+func TestCapGPUOffloadLeavesFittingGPUAlone(t *testing.T) {
+	resolved, engine, variant := baseOffloadInputs()
+	hw := HardwareInfo{GPUVRAMMiB: 49152, UnifiedMemory: false} // 48 GB card fits 7 GB model
+
+	capGPUOffloadForVRAM(resolved, engine, variant, hw)
+
+	if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 999 {
+		t.Errorf("n_gpu_layers = %v, want 999 (untouched) when model fits VRAM", got)
+	}
+}
+
+func TestCapGPUOffloadRespectsUserOverride(t *testing.T) {
+	resolved, engine, variant := baseOffloadInputs()
+	resolved.Config["n_gpu_layers"] = 50
+	resolved.Provenance["n_gpu_layers"] = "L1" // user set it explicitly
+	hw := HardwareInfo{GPUVRAMMiB: 4096, UnifiedMemory: false}
+
+	capGPUOffloadForVRAM(resolved, engine, variant, hw)
+
+	if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 50 {
+		t.Errorf("n_gpu_layers = %v, want 50 (user override respected)", got)
+	}
+}
+
+func TestCapGPUOffloadNoopWithoutOffloadKey(t *testing.T) {
+	resolved, engine, variant := baseOffloadInputs()
+	engine.Amplifier.OffloadConfigKey = "" // engine declares no offload knob
+	hw := HardwareInfo{GPUVRAMMiB: 4096, UnifiedMemory: false}
+
+	capGPUOffloadForVRAM(resolved, engine, variant, hw)
+
+	if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 999 {
+		t.Errorf("n_gpu_layers = %v, want 999 (untouched) when no offload_config_key", got)
+	}
+}
+
+func TestCheckFitSurfacesResolvedWarnings(t *testing.T) {
+	resolved := &ResolvedConfig{
+		Config:   map[string]any{},
+		Warnings: []string{"model needs ~7168 MiB but this GPU has 4096 MiB VRAM"},
+	}
+
+	fit := CheckFit(resolved, HardwareInfo{})
+
+	found := false
+	for _, w := range fit.Warnings {
+		if w == resolved.Warnings[0] {
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("CheckFit did not surface resolve-time warning; got %v", fit.Warnings)
+	}
+}

internal/knowledge/resolver.go

@@ -97,6 +97,10 @@ type ResolvedConfig struct {
 
 	// Performance reference (K4 — historical data or YAML estimate)
 	PerformanceRef *PerformanceReference
+
+	// Warnings collected during resolution (e.g. GPU-offload downgrade). Surfaced
+	// to the user via CheckFit's FitReport so resolve-time decisions are visible.
+	Warnings []string
 }
 
 // PerformanceReference attaches known performance data to a resolved config.
@@ -295,6 +299,7 @@ func (c *Catalog) Resolve(hw HardwareInfo, modelName, engineType string, userOve
 			resolved.EstimatedVRAMMiB = variant.Hardware.VRAMMinMiB
 		}
 		resolved.ResourceEstimate = estimateResources(engine, variant, hw)
+		capGPUOffloadForVRAM(resolved, engine, variant, hw)
 	}
 
 	// Container memory guardrail. On unified-memory hosts (GB10, Apple M-series,
@@ -1596,6 +1601,47 @@ func (c *Catalog) UpsertSyntheticModel(ma ModelAsset) {
 	c.ModelAssets = append(c.ModelAssets, ma)
 }
 
+// capGPUOffloadForVRAM downgrades an engine's GPU-offload knob to CPU when a
+// discrete GPU cannot hold the model. llama.cpp's default n_gpu_layers=999
+// ("offload every layer") assumes the model fits VRAM; on a discrete GPU with
+// less VRAM than the model needs it forces an impossible allocation → CUDA OOM
+// at load, which surfaces to users as a deployment that never becomes ready.
+// Unified-memory hosts (APU/GB10/Apple) share system RAM with the GPU, so full
+// offload is correct there and left untouched. The knob is identified by the
+// engine-declared offload_config_key, so this stays engine-agnostic (INV-1/2):
+// no engine name or config key is hardcoded here.
+func capGPUOffloadForVRAM(resolved *ResolvedConfig, engine *EngineAsset, variant *ModelVariant, hw HardwareInfo) {
+	if resolved == nil || engine == nil || variant == nil {
+		return
+	}
+	key := engine.Amplifier.OffloadConfigKey
+	if key == "" || hw.UnifiedMemory || hw.GPUVRAMMiB <= 0 {
+		return
+	}
+	// Model memory footprint. RAMMinMiB is the catalog's footprint proxy, used
+	// precisely when vram_min_mib=0 (a GGUF variant declared to "fit any GPU"
+	// via CPU offload); fall back to any explicit VRAM estimate.
+	footprint := variant.Hardware.RAMMinMiB
+	if resolved.EstimatedVRAMMiB > footprint {
+		footprint = resolved.EstimatedVRAMMiB
+	}
+	if footprint <= 0 || footprint <= hw.GPUVRAMMiB {
+		return // fits, or footprint unknown — full GPU offload is correct
+	}
+	if resolved.Provenance[key] == "L1" {
+		return // user set the offload knob explicitly — respect it
+	}
+	if cur, ok := resolved.Config[key]; ok && toFloat64(cur) == 0 {
+		return // already CPU-only
+	}
+	resolved.Config[key] = 0
+	resolved.Provenance[key] = "L0-fit"
+	resolved.OffloadPath = true
+	resolved.Warnings = append(resolved.Warnings, fmt.Sprintf(
+		"model needs ~%d MiB but this GPU has %d MiB VRAM; disabling GPU offload and running on CPU (much slower). Use a smaller model or a larger-VRAM GPU for GPU acceleration.",
+		footprint, hw.GPUVRAMMiB))
+}
+
 // estimateResources computes cost(path, R) — the full resource consumption estimate.
 func estimateResources(engine *EngineAsset, variant *ModelVariant, hw HardwareInfo) *ResourceEstimate {
 	perf := variant.ParsedExpectedPerf()
@@ -1644,6 +1690,10 @@ var gmuKeys = []string{"gpu_memory_utilization", "mem_fraction_static"}
 func CheckFit(resolved *ResolvedConfig, hw HardwareInfo) *FitReport {
 	r := &FitReport{Fit: true, Adjustments: make(map[string]any)}
 
+	// Surface warnings collected during resolution (e.g. GPU-offload downgrade)
+	// even if a hard-fail check below returns early.
+	r.Warnings = append(r.Warnings, resolved.Warnings...)
+
 	// Unified memory guard: GPU allocation directly reduces available system memory.
 	// Enforce minimum OS reserve to prevent starvation / swap thrashing.
 	if hw.UnifiedMemory && hw.RAMTotalMiB > 0 {