feat: add catalog overlay diagnostics

AGENTS.md

@@ -182,8 +182,8 @@ catalog/                      # Knowledge assets (go:embed, compiled in)
   models/                     # Model Asset YAML
   partitions/                 # Partition Strategy YAML
   stack/                      # Stack Component YAML (K3S, HAMi — install config + airgap sources)
-# Runtime overlay: ~/.aima/catalog/{hardware,engines,models,partitions,stack}/*.yaml
-#   Same metadata.name overrides go:embed, new names append. No recompilation needed.
+# Runtime overlay: <AIMA_DATA_DIR>/catalog/{central,user}/{hardware,engines,models,partitions,stack,scenarios}/*.patch.yaml
+#   Files are *_patch YAML merged as factory -> central -> user. No recompilation needed.
 ```
 
 ## Key Commands

cmd/aima/deploy_failure.go

@@ -3,10 +3,139 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"strings"
 	"time"
 )
 
+type deploymentErrorCode string
+
+const (
+	deployErrorUnknown              deploymentErrorCode = "UNKNOWN_ERROR"
+	deployErrorOutOfMemory          deploymentErrorCode = "OUT_OF_MEMORY"
+	deployErrorModelNotFound        deploymentErrorCode = "MODEL_NOT_FOUND"
+	deployErrorModelCorrupted       deploymentErrorCode = "MODEL_CORRUPTED"
+	deployErrorModelFormatInvalid   deploymentErrorCode = "MODEL_FORMAT_INVALID"
+	deployErrorPortInUse            deploymentErrorCode = "PORT_IN_USE"
+	deployErrorPermissionDenied     deploymentErrorCode = "PERMISSION_DENIED"
+	deployErrorDownloadFailed       deploymentErrorCode = "DOWNLOAD_FAILED"
+	deployErrorHardwareIncompatible deploymentErrorCode = "HARDWARE_INCOMPATIBLE"
+	deployErrorTimeout              deploymentErrorCode = "TIMEOUT"
+	deployErrorEngineStartFailed    deploymentErrorCode = "ENGINE_START_FAILED"
+)
+
+type deploymentCleanupResult struct {
+	Attempted bool   `json:"attempted"`
+	Succeeded bool   `json:"succeeded"`
+	Message   string `json:"message"`
+}
+
+type deploymentRunError struct {
+	Code    deploymentErrorCode
+	Message string
+	Cleanup deploymentCleanupResult
+}
+
+func (e deploymentRunError) Error() string {
+	msg := fmt.Sprintf("%s: %s", e.Code, e.Message)
+	if e.Cleanup.Message != "" {
+		msg += "; cleanup: " + e.Cleanup.Message
+	}
+	return msg
+}
+
+func newDeploymentRunError(code deploymentErrorCode, message string, cleanup deploymentCleanupResult) error {
+	if code == "" {
+		code = deployErrorUnknown
+	}
+	return deploymentRunError{Code: code, Message: strings.TrimSpace(message), Cleanup: cleanup}
+}
+
+func cleanupFailedDeployment(ctx context.Context, deployName string, deleteFn func(context.Context, string) error) deploymentCleanupResult {
+	deployName = strings.TrimSpace(deployName)
+	if deployName == "" {
+		return deploymentCleanupResult{Message: "deployment name unavailable"}
+	}
+	if deleteFn == nil {
+		return deploymentCleanupResult{Message: "deploy.delete unavailable"}
+	}
+	if err := deleteFn(ctx, deployName); err != nil {
+		return deploymentCleanupResult{
+			Attempted: true,
+			Succeeded: false,
+			Message:   "delete failed deployment " + deployName + ": " + err.Error(),
+		}
+	}
+	return deploymentCleanupResult{
+		Attempted: true,
+		Succeeded: true,
+		Message:   "deleted failed deployment " + deployName,
+	}
+}
+
+func classifyDeploymentFailure(message string) deploymentErrorCode {
+	lower := strings.ToLower(strings.TrimSpace(message))
+	switch {
+	case lower == "":
+		return deployErrorUnknown
+	case strings.Contains(lower, "outofmemoryerror"),
+		strings.Contains(lower, "out of memory"),
+		strings.Contains(lower, "oom"),
+		strings.Contains(lower, "cuda error: out of memory"),
+		strings.Contains(lower, "hip out of memory"):
+		return deployErrorOutOfMemory
+	case strings.Contains(lower, "address already in use"),
+		strings.Contains(lower, "bind: address"),
+		strings.Contains(lower, "port is in use"),
+		strings.Contains(lower, "port already allocated"):
+		return deployErrorPortInUse
+	case strings.Contains(lower, "permission denied"),
+		strings.Contains(lower, "operation not permitted"),
+		strings.Contains(lower, "access is denied"):
+		return deployErrorPermissionDenied
+	case strings.Contains(lower, "not ready within"),
+		strings.Contains(lower, "timed out"),
+		strings.Contains(lower, "timeout"):
+		return deployErrorTimeout
+	case strings.Contains(lower, "corrupt"),
+		strings.Contains(lower, "checksum"),
+		strings.Contains(lower, "incomplete"),
+		strings.Contains(lower, "safetensorerror"):
+		return deployErrorModelCorrupted
+	case strings.Contains(lower, "filenotfounderror"),
+		strings.Contains(lower, "no such file"),
+		strings.Contains(lower, "model not found"),
+		strings.Contains(lower, "cannot find model"),
+		strings.Contains(lower, "not found"):
+		return deployErrorModelNotFound
+	case strings.Contains(lower, "invalid model"),
+		strings.Contains(lower, "unsupported model"),
+		strings.Contains(lower, "unknown architecture"),
+		strings.Contains(lower, "invalid file magic"),
+		strings.Contains(lower, "invalid gguf"),
+		strings.Contains(lower, "safetensors header"):
+		return deployErrorModelFormatInvalid
+	case strings.Contains(lower, "download"),
+		strings.Contains(lower, "pull"),
+		strings.Contains(lower, "connection reset"),
+		strings.Contains(lower, "tls handshake"),
+		strings.Contains(lower, "temporary failure in name resolution"):
+		return deployErrorDownloadFailed
+	case strings.Contains(lower, "hardware not compatible"),
+		strings.Contains(lower, "not compatible"),
+		strings.Contains(lower, "compute capability"),
+		strings.Contains(lower, "no suitable device"):
+		return deployErrorHardwareIncompatible
+	case strings.Contains(lower, "process exited"),
+		strings.Contains(lower, "startup"),
+		strings.Contains(lower, "failed core proc"),
+		strings.Contains(lower, "failed"):
+		return deployErrorEngineStartFailed
+	default:
+		return deployErrorUnknown
+	}
+}
+
 type deploymentFailureDetails struct {
 	Message        string
 	StartupMessage string

cmd/aima/main.go

@@ -1357,7 +1357,17 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps {
 			}
 		}
 
-		waitForDeployment := func(deployName, runtimeName, resolvedEngine string, resolvedConfig map[string]any, warmup knowledge.WarmupConfig, deployTimeout time.Duration) (json.RawMessage, error) {
+		waitForDeployment := func(deployName, runtimeName, resolvedEngine string, resolvedConfig map[string]any, warmup knowledge.WarmupConfig, deployTimeout time.Duration, cleanupOnFailure bool) (json.RawMessage, error) {
+			cleanup := func() deploymentCleanupResult {
+				if !cleanupOnFailure {
+					return deploymentCleanupResult{}
+				}
+				result := cleanupFailedDeployment(ctx, deployName, deps.DeployDelete)
+				if result.Attempted {
+					notify("cleanup", result.Message)
+				}
+				return result
+			}
 			notify("waiting", deployName)
 			ticker := time.NewTicker(2 * time.Second)
 			defer ticker.Stop()
@@ -1376,10 +1386,8 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps {
 				case <-ctx.Done():
 					return nil, ctx.Err()
 				case <-timer.C:
-					return json.Marshal(map[string]any{
-						"name": deployName, "status": "timeout",
-						"message": fmt.Sprintf("deployment started but not ready within %s", deployTimeout),
-					})
+					msg := fmt.Sprintf("deployment started but not ready within %s", deployTimeout)
+					return nil, newDeploymentRunError(deployErrorTimeout, msg, cleanup())
 				case <-ticker.C:
 					statusData, err := deps.DeployStatus(ctx, deployName)
 					if err != nil {
@@ -1420,7 +1428,7 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps {
 							StartupMessage: status.StartupMessage,
 							ErrorLines:     status.ErrorLines,
 						}, deps.DeployStatus, deps.DeployLogs)
-						return nil, fmt.Errorf("deployment failed: %s", msg)
+						return nil, newDeploymentRunError(classifyDeploymentFailure(msg), "deployment failed: "+msg, cleanup())
 					}
 					phase := status.StartupPhase
 					if phase == "" {
@@ -1458,7 +1466,7 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps {
 			return nil, fmt.Errorf("parse resolve result: %w", err)
 		}
 		if !plan.FitReport.Fit {
-			return nil, fmt.Errorf("hardware not compatible: %s", plan.FitReport.Reason)
+			return nil, newDeploymentRunError(deployErrorHardwareIncompatible, "hardware not compatible: "+plan.FitReport.Reason, deploymentCleanupResult{})
 		}
 		notify("resolved", fmt.Sprintf("engine=%s runtime=%s", plan.Engine, plan.Runtime))
 		for _, warn := range plan.FitReport.Warns {
@@ -1501,7 +1509,7 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps {
 							deployTimeout = time.Duration(t) * time.Second
 						}
 					}
-					return waitForDeployment(deployName, runtimeName, plan.Engine, plan.Config, warmup, deployTimeout)
+					return waitForDeployment(deployName, runtimeName, plan.Engine, plan.Config, warmup, deployTimeout, false)
 				}
 			}
 		}
@@ -1538,6 +1546,7 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps {
 		var deployResult struct {
 			Name    string `json:"name"`
 			Runtime string `json:"runtime"`
+			Reused  bool   `json:"reused"`
 		}
 		if err := json.Unmarshal(deployData, &deployResult); err != nil || deployResult.Name == "" {
 			return deployData, nil
@@ -1551,7 +1560,7 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps {
 				deployTimeout = time.Duration(t) * time.Second
 			}
 		}
-		return waitForDeployment(deployResult.Name, deployResult.Runtime, plan.Engine, plan.Config, warmup, deployTimeout)
+		return waitForDeployment(deployResult.Name, deployResult.Runtime, plan.Engine, plan.Config, warmup, deployTimeout, !deployResult.Reused)
 	}
 
 	deps = &mcp.ToolDeps{}

cmd/aima/main_test.go

@@ -1276,6 +1276,48 @@ func TestApplyScenarioWaitsOnLastStepBeforePostDeploy(t *testing.T) {
 	}
 }
 
+func TestClassifyDeploymentFailure(t *testing.T) {
+	tests := []struct {
+		name string
+		msg  string
+		want deploymentErrorCode
+	}{
+		{name: "out of memory", msg: "RuntimeError: HIP out of memory while loading weights", want: deployErrorOutOfMemory},
+		{name: "model corrupted", msg: "safetensors_rust.SafetensorError: Error while deserializing header: incomplete metadata", want: deployErrorModelCorrupted},
+		{name: "model not found", msg: "FileNotFoundError: no such file or directory: /models/demo/config.json", want: deployErrorModelNotFound},
+		{name: "port occupied", msg: "bind: address already in use on 0.0.0.0:8000", want: deployErrorPortInUse},
+		{name: "permission", msg: "permission denied opening /dev/kfd", want: deployErrorPermissionDenied},
+		{name: "timeout", msg: "deployment started but not ready within 30s", want: deployErrorTimeout},
+		{name: "generic startup", msg: "process exited before readiness", want: deployErrorEngineStartFailed},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := classifyDeploymentFailure(tt.msg); got != tt.want {
+				t.Fatalf("classifyDeploymentFailure(%q) = %q, want %q", tt.msg, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestCleanupFailedDeploymentReportsResult(t *testing.T) {
+	var deleted string
+	result := cleanupFailedDeployment(context.Background(), "demo", func(ctx context.Context, name string) error {
+		deleted = name
+		return nil
+	})
+	if deleted != "demo" {
+		t.Fatalf("deleted deployment = %q, want demo", deleted)
+	}
+	if !result.Attempted || !result.Succeeded || result.Message != "deleted failed deployment demo" {
+		t.Fatalf("cleanup result = %#v, want successful deletion", result)
+	}
+
+	skipped := cleanupFailedDeployment(context.Background(), "demo", nil)
+	if skipped.Attempted || skipped.Succeeded || !strings.Contains(skipped.Message, "unavailable") {
+		t.Fatalf("skipped cleanup result = %#v, want unavailable message", skipped)
+	}
+}
+
 func TestVariantQuantizationHint(t *testing.T) {
 	if got := variantQuantizationHint(&knowledge.ModelVariant{
 		DefaultConfig: map[string]any{"quantization": "gptq"},