From b0ef4c642206d8ca4a21a4fc5dc619b9486ead2e Mon Sep 17 00:00:00 2001 From: homen Date: Tue, 21 Jul 2026 21:31:16 -0700 Subject: [PATCH 01/25] feat: add CreatorBench v1 governance and routing --- apps/atlas/src/atlas.tsx | 3 + apps/creatorbench/src/creatorbench.css | 1529 +++++++++++++++++ apps/creatorbench/src/creatorbench.tsx | 1232 +++++++++++++ apps/edit/src/creator-workspace.tsx | 3 + .../catalog/public-instances.json | 431 +++++ .../catalog/public-sources.json | 38 + .../catalog/public-splits.json | 18 + .../creatorbench-v1/config/domains.json | 65 + .../config/routing-policy.json | 20 + .../receipts/acquisition-receipt.json | 30 + .../receipts/deduplication-receipt.json | 14 + .../receipts/instance-receipt.json | 18 + .../results/public-evaluation.json | 747 ++++++++ .../schemas/creator-request.schema.json | 203 +++ .../schemas/freeze.schema.json | 64 + .../schemas/instance.schema.json | 52 + .../schemas/public-claim.schema.json | 80 + .../schemas/result.schema.json | 96 ++ .../schemas/reviewer.schema.json | 49 + .../schemas/source.schema.json | 87 + .../creatorbench-v1/schemas/split.schema.json | 29 + biome.json | 3 + config/executor-routing/default-policy.json | 15 + .../executor-routing/executor-profiles.json | 24 + convex/creatorbenchReview.ts | 132 ++ convex/schema.ts | 34 + creatorbench.html | 17 + docs/benchmarks/CREATORBENCH_V1_GOVERNANCE.md | 109 ++ docs/benchmarks/PRODUCTION_SHADOW_MODE.md | 5 + docs/benchmarks/PUBLIC_CLAIM_POLICY.md | 41 + docs/benchmarks/SEALED_EVALUATION.md | 12 + docs/testing/CREATORBENCH_E2E_TEST_PROMPTS.md | 75 + package.json | 13 + scripts/benchmarks/acquire-creatorbench.mjs | 558 ++++++ .../build-creatorbench-manifests.mjs | 156 ++ scripts/benchmarks/creatorbench-io.mjs | 44 + .../benchmarks/deduplicate-creatorbench.mjs | 114 ++ scripts/benchmarks/evaluate-creatorbench.mjs | 293 ++++ scripts/benchmarks/freeze-creatorbench.mjs | 81 + .../publish-creatorbench-report.mjs | 99 ++ scripts/benchmarks/validate-creatorbench.mjs | 80 + src/lib/adaptive-executor-router.test.ts | 192 +++ src/lib/adaptive-executor-router.ts | 345 ++++ src/lib/creatorbench-contracts.test.ts | 374 ++++ src/lib/creatorbench-contracts.ts | 713 ++++++++ src/lib/creatorbench-evaluators.test.ts | 40 + src/lib/creatorbench-evaluators.ts | 126 ++ src/lib/creatorbench-shadow.test.ts | 40 + src/lib/creatorbench-shadow.ts | 99 ++ src/lib/creatorbench-statistics.test.ts | 15 + src/lib/creatorbench-statistics.ts | 62 + src/lib/route-receipts.ts | 138 ++ tests/e2e/creatorbench.spec.ts | 241 +++ vercel.json | 1 + vite.config.ts | 23 +- 55 files changed, 9121 insertions(+), 1 deletion(-) create mode 100644 apps/creatorbench/src/creatorbench.css create mode 100644 apps/creatorbench/src/creatorbench.tsx create mode 100644 benchmarks/creatorbench-v1/catalog/public-instances.json create mode 100644 benchmarks/creatorbench-v1/catalog/public-sources.json create mode 100644 benchmarks/creatorbench-v1/catalog/public-splits.json create mode 100644 benchmarks/creatorbench-v1/config/domains.json create mode 100644 benchmarks/creatorbench-v1/config/routing-policy.json create mode 100644 benchmarks/creatorbench-v1/receipts/acquisition-receipt.json create mode 100644 benchmarks/creatorbench-v1/receipts/deduplication-receipt.json create mode 100644 benchmarks/creatorbench-v1/receipts/instance-receipt.json create mode 100644 benchmarks/creatorbench-v1/results/public-evaluation.json create mode 100644 benchmarks/creatorbench-v1/schemas/creator-request.schema.json create mode 100644 benchmarks/creatorbench-v1/schemas/freeze.schema.json create mode 100644 benchmarks/creatorbench-v1/schemas/instance.schema.json create mode 100644 benchmarks/creatorbench-v1/schemas/public-claim.schema.json create mode 100644 benchmarks/creatorbench-v1/schemas/result.schema.json create mode 100644 benchmarks/creatorbench-v1/schemas/reviewer.schema.json create mode 100644 benchmarks/creatorbench-v1/schemas/source.schema.json create mode 100644 benchmarks/creatorbench-v1/schemas/split.schema.json create mode 100644 config/executor-routing/default-policy.json create mode 100644 config/executor-routing/executor-profiles.json create mode 100644 convex/creatorbenchReview.ts create mode 100644 creatorbench.html create mode 100644 docs/benchmarks/CREATORBENCH_V1_GOVERNANCE.md create mode 100644 docs/benchmarks/PRODUCTION_SHADOW_MODE.md create mode 100644 docs/benchmarks/PUBLIC_CLAIM_POLICY.md create mode 100644 docs/benchmarks/SEALED_EVALUATION.md create mode 100644 docs/testing/CREATORBENCH_E2E_TEST_PROMPTS.md create mode 100644 scripts/benchmarks/acquire-creatorbench.mjs create mode 100644 scripts/benchmarks/build-creatorbench-manifests.mjs create mode 100644 scripts/benchmarks/creatorbench-io.mjs create mode 100644 scripts/benchmarks/deduplicate-creatorbench.mjs create mode 100644 scripts/benchmarks/evaluate-creatorbench.mjs create mode 100644 scripts/benchmarks/freeze-creatorbench.mjs create mode 100644 scripts/benchmarks/publish-creatorbench-report.mjs create mode 100644 scripts/benchmarks/validate-creatorbench.mjs create mode 100644 src/lib/adaptive-executor-router.test.ts create mode 100644 src/lib/adaptive-executor-router.ts create mode 100644 src/lib/creatorbench-contracts.test.ts create mode 100644 src/lib/creatorbench-contracts.ts create mode 100644 src/lib/creatorbench-evaluators.test.ts create mode 100644 src/lib/creatorbench-evaluators.ts create mode 100644 src/lib/creatorbench-shadow.test.ts create mode 100644 src/lib/creatorbench-shadow.ts create mode 100644 src/lib/creatorbench-statistics.test.ts create mode 100644 src/lib/creatorbench-statistics.ts create mode 100644 src/lib/route-receipts.ts create mode 100644 tests/e2e/creatorbench.spec.ts diff --git a/apps/atlas/src/atlas.tsx b/apps/atlas/src/atlas.tsx index d0184b7..28c84e2 100644 --- a/apps/atlas/src/atlas.tsx +++ b/apps/atlas/src/atlas.tsx @@ -459,6 +459,9 @@ function AtlasApp() { Creator workspace + + CreatorBench +

Explore

{MODES.map(({ id, label, icon: Icon }) => (