fix: tier 1 correctness and cleanup fixes from architecture audit

package.json

@@ -25,10 +25,6 @@
     "test:e2e:ui": "playwright test --ui",
     "test:e2e:report": "playwright show-report",
     "pr:screenshots": "node scripts/pr-screenshots.mjs",
-    "test:personality": "tsx src/cli/personality-test.ts",
-    "compare:personality": "tsx src/cli/compare-personality.ts",
-    "test:preference": "tsx src/cli/preference-test.ts",
-    "test:self-preference": "tsx src/cli/self-preference-test.ts",
     "cli": "tsx --no-cache -r ./src/cli/env-loader.js src/cli/index.ts",
     "cli:generate-pain-points": "pnpm cli generate-pain-points",
     "cli:annotate-pain-points": "pnpm cli annotate-pain-points",

src/app/analysis/[configId]/[runLabel]/[timestamp]/ClientPage.tsx

@@ -25,7 +25,7 @@ import { BLUEPRINT_CONFIG_REPO_URL } from '@/lib/configConstants';
 import { SinglePromptView } from '@/app/analysis/components/SinglePromptView';
 import { AggregateAnalysisView } from '@/app/analysis/components/AggregateAnalysisView';
 import { useToast } from '@/components/ui/use-toast';
-import { generateMinimalBlueprintYaml } from '@/app/sandbox/utils/yaml-generator';
+import { generateMinimalBlueprintYaml } from '@/lib/yaml-generator';
 import { useAnalysis } from '@/app/analysis/context/AnalysisContext';
 import AnalysisPageHeader from '@/app/analysis/components/AnalysisPageHeader';
 import { fromSafeTimestamp, formatTimestampForDisplay } from '@/lib/timestampUtils';

src/app/api/sandbox/auto-create/route.ts

@@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from 'next/server';
 import { LLMApiCallOptions } from '@/lib/llm-clients/types';
 import { getModelResponse } from '@/cli/services/llm-service';
 import { checkForErrors } from '@/cli/utils/response-utils';
-import { parseWevalConfigFromResponse } from '@/app/sandbox/utils/json-response-parser';
+import { parseWevalConfigFromResponse } from '@/lib/json-response-parser';
 import { fromZodError } from 'zod-validation-error';
 import { z } from 'zod';
 import {

src/app/api/sandbox/auto-extend/route.ts

@@ -1,8 +1,8 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { getModelResponse } from '@/cli/services/llm-service';
 import { checkForErrors } from '@/cli/utils/response-utils';
-import { parsePromptsFromResponse } from '@/app/sandbox/utils/json-response-parser';
-import { generateMinimalBlueprintYaml } from '@/app/sandbox/utils/yaml-generator';
+import { parsePromptsFromResponse } from '@/lib/json-response-parser';
+import { generateMinimalBlueprintYaml } from '@/lib/yaml-generator';
 import { WevalConfig } from '@/types/shared';
 import { z } from 'zod';
 import {

src/app/api/sandbox/auto-wiki/route.ts

@@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from 'next/server';
 import { z } from 'zod';
 import { getModelResponse } from '@/cli/services/llm-service';
 import { checkForErrors } from '@/cli/utils/response-utils';
-import { parseWevalConfigFromResponse } from '@/app/sandbox/utils/json-response-parser';
+import { parseWevalConfigFromResponse } from '@/lib/json-response-parser';
 import { fromZodError } from 'zod-validation-error';
 import {
     EXPERT_PREAMBLE,

src/app/api/story/__tests__/json-response-parser.test.ts

@@ -1,5 +1,5 @@
 import { vi } from 'vitest';
-import { parseWevalConfigFromResponse } from '@/app/sandbox/utils/json-response-parser';
+import { parseWevalConfigFromResponse } from '@/lib/json-response-parser';
 
 // Mock the LLM service
 vi.mock('@/cli/services/llm-service', () => ({
@@ -10,7 +10,7 @@ vi.mock('@/cli/utils/response-utils', () => ({
   checkForErrors: vi.fn(() => false)
 }));
 
-vi.mock('@/app/sandbox/utils/yaml-generator', () => ({
+vi.mock('@/lib/yaml-generator', () => ({
   generateMinimalBlueprintYaml: vi.fn((config) => `title: "${config.title}"\ndescription: "${config.description}"`)
 }));
 

src/app/api/story/create/route.ts

@@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from 'next/server';
 import { configure } from '@/cli/config';
 import { getLogger } from '@/utils/logger';
 import { CREATOR_SYSTEM_PROMPT } from '../utils/prompt-constants';
-import { parseWevalConfigFromResponse } from '@/app/sandbox/utils/json-response-parser';
+import { parseWevalConfigFromResponse } from '@/lib/json-response-parser';
 import { resilientLLMCall, validateStoryResponse } from '../utils/llm-resilience';
 import { z } from 'zod';
 

src/app/api/story/export/__tests__/route.test.ts

@@ -15,7 +15,7 @@ vi.mock('@/utils/logger', () => ({
     error: vi.fn(),
   })),
 }));
-vi.mock('@/app/sandbox/utils/yaml-generator', () => ({
+vi.mock('@/lib/yaml-generator', () => ({
   generateMinimalBlueprintYaml: vi.fn((obj) => `# Generated YAML\ntitle: ${obj.title || 'Test'}`),
 }));
 

src/app/api/story/export/route.ts

@@ -1,6 +1,6 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { saveJsonFile, getJsonFile } from '@/lib/storageService';
-import { generateMinimalBlueprintYaml } from '@/app/sandbox/utils/yaml-generator';
+import { generateMinimalBlueprintYaml } from '@/lib/yaml-generator';
 import { getLogger } from '@/utils/logger';
 import { z } from 'zod';
 import type { ComparisonConfig } from '@/cli/types/cli_types';

src/app/api/story/update/route.ts

@@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from 'next/server';
 import { configure } from '@/cli/config';
 import { getLogger } from '@/utils/logger';
 import { UPDATER_SYSTEM_PROMPT } from '../utils/prompt-constants';
-import { parseWevalConfigFromResponse } from '@/app/sandbox/utils/json-response-parser';
+import { parseWevalConfigFromResponse } from '@/lib/json-response-parser';
 import { resilientLLMCall, validateStoryResponse } from '../utils/llm-resilience';
 import { z } from 'zod';
 

src/app/api/webhooks/github-pr/route.ts

@@ -6,7 +6,11 @@ import { callBackgroundFunction } from '@/lib/background-function-client';
 import { prEvaluationLimiter, webhookIPLimiter, webhookGlobalLimiter } from '@/lib/webhook-rate-limiter';
 import { checkPREvalLimits, formatLimitViolations, PR_EVAL_LIMITS } from '@/lib/pr-eval-limiter';
 import { validateReservedPrefixes } from '@/lib/blueprint-parser';
+import { validateBlueprintSchema } from '@/lib/blueprint-validator';
+import { formatSchemaErrors } from '@/lib/blueprint-ingestion';
 import { generateBlueprintIdFromPath, validateBlueprintId } from '@/app/utils/blueprintIdUtils';
+import type { ComparisonConfig } from '@/cli/types/cli_types';
+import type { ErrorObject } from 'ajv';
 
 const GITHUB_WEBHOOK_SECRET = process.env.GITHUB_WEBHOOK_SECRET;
 const UPSTREAM_OWNER = 'weval-org';
@@ -432,8 +436,25 @@ export async function POST(req: NextRequest) {
 
       // Parse blueprint for limit checking using the proper parser
       // This handles all multi-document formats correctly
-      const { parseAndNormalizeBlueprint } = await import('@/lib/blueprint-parser');
-      const config = parseAndNormalizeBlueprint(content, 'yaml');
+      let config: ComparisonConfig;
+      try {
+        const { parseAndNormalizeBlueprint } = await import('@/lib/blueprint-parser');
+        config = parseAndNormalizeBlueprint(content, 'yaml');
+      } catch (parseError: any) {
+        validationErrors.push({ filename: file.filename, error: `Parse error: ${parseError.message}` });
+        continue;
+      }
+
+      // Fail fast on canonical-schema violations so contributors get
+      // actionable feedback at PR time instead of a burned staging run.
+      const schemaCheck = validateBlueprintSchema(config, 'canonical');
+      if (!schemaCheck.valid) {
+        validationErrors.push({
+          filename: file.filename,
+          error: `Schema validation failed: ${formatSchemaErrors(schemaCheck.errors as ErrorObject[] | null)}`,
+        });
+        continue;
+      }
 
       // Check PR evaluation limits
       // Note: checkPREvalLimits doesn't need auth for public model collections

src/app/api/webhooks/github-push/route.ts

@@ -4,6 +4,9 @@ import crypto from 'crypto';
 import { callBackgroundFunction } from '@/lib/background-function-client';
 import { webhookIPLimiter, webhookGlobalLimiter } from '@/lib/webhook-rate-limiter';
 import { generateConfigContentHash } from '@/lib/hash-utils';
+import { parseSubmittedBlueprint } from '@/lib/blueprint-ingestion';
+import { resolveModelsInConfig } from '@/lib/blueprint-service';
+import { ComparisonConfig } from '@/cli/types/cli_types';
 import { S3Client, HeadObjectCommand } from '@aws-sdk/client-s3';
 
 const GITHUB_WEBHOOK_SECRET = process.env.GITHUB_WEBHOOK_SECRET;
@@ -155,35 +158,6 @@ async function hasBeenEvaluated(configId: string, contentHash: string): Promise<
   }
 }
 
-/**
- * Parse and validate blueprint
- */
-async function parseBlueprint(content: string): Promise<any | null> {
-  try {
-    const yaml = await import('js-yaml');
-    const parsed = yaml.load(content);
-
-    if (!parsed || typeof parsed !== 'object') {
-      return null;
-    }
-
-    const config = parsed as any;
-
-    if (!config.id || typeof config.id !== 'string') {
-      return null;
-    }
-
-    if (!config.prompts || !Array.isArray(config.prompts) || config.prompts.length === 0) {
-      return null;
-    }
-
-    return config;
-  } catch (error: any) {
-    console.error('[GitHub Push Webhook] Parse error:', error.message);
-    return null;
-  }
-}
-
 /**
  * Trigger production evaluation
  */
@@ -334,20 +308,40 @@ export async function POST(req: NextRequest) {
         continue;
       }
 
-      // Parse blueprint
-      const config = await parseBlueprint(content);
-      if (!config) {
-        console.error(`[GitHub Push Webhook] Invalid blueprint: ${file.filename}`);
-        results.errors.push(`${file.filename}: Invalid blueprint structure`);
+      // Parse with the canonical parser (multi-document YAML, path-derived ID,
+      // canonical schema validation) so merged blueprints behave identically
+      // to the PR-staging and scheduled-eval ingestion paths.
+      const parsed = parseSubmittedBlueprint(content, file.filename);
+      for (const warning of parsed.warnings) {
+        console.warn(`[GitHub Push Webhook] ${file.filename}: ${warning}`);
+      }
+      if (!parsed.config) {
+        console.error(`[GitHub Push Webhook] Invalid blueprint ${file.filename}: ${parsed.error}`);
+        results.errors.push(`${file.filename}: ${parsed.error}`);
         continue;
       }
 
-      // Calculate content hash
-      const modelIds = config.models?.map((m: any) => typeof m === 'string' ? m : m.id) || [];
-      const contentHash = generateConfigContentHash({ ...config, models: modelIds });
+      // Resolve model collections (e.g. CORE) before hashing so the dedupe
+      // hash matches the scheduled-eval path, which hashes the resolved config.
+      let config: ComparisonConfig;
+      try {
+        config = await resolveModelsInConfig(parsed.config, GITHUB_TOKEN);
+      } catch (error: any) {
+        results.errors.push(`${file.filename}: Failed to resolve model collections: ${error.message}`);
+        continue;
+      }
+      if (!config.models || config.models.length === 0) {
+        results.errors.push(`${file.filename}: No models remain after collection resolution`);
+        continue;
+      }
+
+      const contentHash = generateConfigContentHash(config);
+
+      // parseSubmittedBlueprint always sets a path-derived id
+      const configId = config.id!;
 
       // Check if already evaluated
-      const alreadyEvaluated = await hasBeenEvaluated(config.id, contentHash);
+      const alreadyEvaluated = await hasBeenEvaluated(configId, contentHash);
 
       if (alreadyEvaluated) {
         console.log(`[GitHub Push Webhook] Blueprint ${config.id} already evaluated (hash: ${contentHash})`);