From 40be75cf2db61ea4eacd98d63ed92e97bd0642ba Mon Sep 17 00:00:00 2001 From: phroi <90913182+phroi@users.noreply.github.com> Date: Thu, 28 May 2026 19:44:18 +0000 Subject: [PATCH 1/7] feat(supervisor): capture live evidence summaries --- apps/supervisor/src/index.test.ts | 1485 ++++++++++++++++++++++++++--- apps/supervisor/src/index.ts | 593 +++++++++--- 2 files changed, 1836 insertions(+), 242 deletions(-) diff --git a/apps/supervisor/src/index.test.ts b/apps/supervisor/src/index.test.ts index 011bcbd..3d3e3bf 100644 --- a/apps/supervisor/src/index.test.ts +++ b/apps/supervisor/src/index.test.ts @@ -13,7 +13,6 @@ import { recordScenarioAttempt, recordOutcome, resolvePlan, - safeArtifactText, supervise, usage, type CommandResult, @@ -62,6 +61,9 @@ describe("supervisor CLI", () => { expect(parseArgs(["--tester-scenario", "two-ickb-to-ckb-limit-orders"]).testerScenario).toBe( "two-ickb-to-ckb-limit-orders", ); + expect(parseArgs(["--tester-scenario", "bounded-ickb-to-ckb-limit-order"]).testerScenario).toBe( + "bounded-ickb-to-ckb-limit-order", + ); expect(parseArgs(["--tester-scenario", "mixed-direction-limit-orders"]).testerScenario).toBe( "mixed-direction-limit-orders", ); @@ -103,7 +105,7 @@ describe("supervisor CLI", () => { const args = parseArgs(["--dry-run", "--out-dir", "not-ignored"]); expect(() => resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(false) })).toThrow( - "Supervisor output directory must be under logs/live-supervisor/", + "Supervisor output directory must be under logs/live-supervisor/ or a validation session run directory", ); }); @@ -111,7 +113,7 @@ describe("supervisor CLI", () => { const args = parseArgs(["--dry-run", "--out-dir", "config/supervisor"]); expect(() => resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) })).toThrow( - "Supervisor output directory must be under logs/live-supervisor/", + "Supervisor output directory must be under logs/live-supervisor/ or a validation session run directory", ); }); @@ -124,6 +126,50 @@ describe("supervisor CLI", () => { expect(plan.testerConfigPath).toBeUndefined(); }); + it("accepts dynamic validation session artifact paths", () => { + const args = parseArgs(["--dry-run", "--out-dir", "log/validation/dynamic-test/chunks/chunk-0001/run-0001"]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set(["log/validation/dynamic-test/chunks/chunk-0001/run-0001"])) }); + + expect(plan.relativeOutDir).toBe("log/validation/dynamic-test/chunks/chunk-0001/run-0001"); + expect(plan.botConfigPath).toBeUndefined(); + expect(plan.testerConfigPath).toBeUndefined(); + }); + + it("rejects validation roots outside run artifact directories", () => { + const args = parseArgs(["--dry-run", "--out-dir", "log/validation/dynamic-test"]); + + expect(() => resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) })).toThrow( + "Supervisor output directory must be under logs/live-supervisor/ or a validation session run directory", + ); + }); + + it("rejects validation run artifact directory descendants", () => { + const args = parseArgs(["--dry-run", "--out-dir", "log/validation/dynamic-test/chunks/chunk-0001/run-0001/extra"]); + + expect(() => resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) })).toThrow( + "Supervisor output directory must be under logs/live-supervisor/ or a validation session run directory", + ); + }); + + it("accepts explicit validation session roots outside the repo", () => { + const args = parseArgs(["--dry-run", "--out-dir", "/var/tmp/ickb-log/validation/dynamic-test/chunks/chunk-0001/run-0001"]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(false) }); + + expect(plan.relativeOutDir).toBe("/var/tmp/ickb-log/validation/dynamic-test/chunks/chunk-0001/run-0001"); + expect(plan.outDir).toBe("/var/tmp/ickb-log/validation/dynamic-test/chunks/chunk-0001/run-0001"); + }); + + it("refuses symlinked explicit validation parents outside the repo", async () => { + const args = parseArgs(["--dry-run", "--out-dir", "/var/tmp/ickb-log/validation/dynamic-test/chunks/chunk-0001/run-0001"]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(false) }); + + await expect(supervise(args, plan, { + lstat: (path) => Promise.resolve({ isSymbolicLink: () => pathToString(path) === "/var/tmp/ickb-log/validation" } as never), + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + })).rejects.toThrow("Refusing to write supervisor artifacts through symlinked path: /var/tmp/ickb-log/validation"); + }); + it("resolves default ignored live config paths", () => { const plan = resolvePlan(parseArgs(["--out-dir", "logs/live-supervisor/test"]), "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ "logs/live-supervisor/test", @@ -206,7 +252,8 @@ describe("supervisor CLI", () => { const actor = spawned.find((item) => item.args[0] === "apps/bot/dist/index.js"); expect(preflight?.env).not.toHaveProperty("PRIVATE_KEY"); expect(preflight?.env).not.toHaveProperty("COWORKER_BUILD"); - expect(actor?.env).toMatchObject({ BOT_CONFIG_FILE: "/repo/config/bot-testnet.json", INIT_CWD: "/repo" }); + expect(preflight?.env).toMatchObject({ INIT_CWD: "/repo", NODE_OPTIONS: "--disable-warning=DEP0040" }); + expect(actor?.env).toMatchObject({ BOT_CONFIG_FILE: "/repo/config/bot-testnet.json", INIT_CWD: "/repo", NODE_OPTIONS: "--disable-warning=DEP0040" }); expect(actor?.env).not.toHaveProperty("PRIVATE_KEY"); expect(actor?.env).not.toHaveProperty("COWORKER_BUILD"); } finally { @@ -372,7 +419,7 @@ describe("supervisor CLI", () => { expect(bot?.env).not.toHaveProperty("TESTER_FEE_BASE"); }); - it("runs the same tester twice for fresh-skip multi-order coverage", async () => { + it("runs the same tester twice for fresh-skip auto coverage", async () => { const spawned: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []; const writes = new Map(); let testerRuns = 0; @@ -397,13 +444,9 @@ describe("supervisor CLI", () => { ? { startTime: "now", actions: { - requestedTesterScenario: "multi-order-limit-orders", - testerScenario: "mixed-direction-limit-orders", - newOrders: [ - { giveCkb: "10", takeIckb: "9", fee: "0.1" }, - { giveIckb: "20", takeCkb: "18", fee: "0.2" }, - ], - orderCount: 2, + requestedTesterScenario: "auto", + testerScenario: "bounded-ickb-to-ckb-limit-order", + newOrder: { giveIckb: "20", takeCkb: "18", fee: "0.2" }, cancelledOrders: 0, }, txHash: txHash("77"), @@ -424,7 +467,7 @@ describe("supervisor CLI", () => { const testerSpawns = spawned.filter((item) => item.args[0] === "apps/tester/dist/index.js"); expect(exitCode).toBe(0); expect(testerSpawns).toHaveLength(2); - expect(testerSpawns[0]?.env).toMatchObject({ TESTER_SCENARIO: "multi-order-limit-orders" }); + expect(testerSpawns[0]?.env).toMatchObject({ TESTER_SCENARIO: "auto" }); expect(testerSpawns[1]?.env).toMatchObject({ TESTER_SCENARIO: "auto" }); expect(writes.has("/repo/logs/live-supervisor/two-pass-test/cycle-0001-tester-pass-1.stdout.ndjson")).toBe(true); expect(writes.has("/repo/logs/live-supervisor/two-pass-test/cycle-0001-tester-pass-2.stdout.ndjson")).toBe(true); @@ -435,6 +478,211 @@ describe("supervisor CLI", () => { }); }); + it("uses tester auto for low-CKB first-pass fresh-skip fundability", async () => { + const spawned: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []; + let testerRuns = 0; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/two-pass-low-ckb-test", + "--scenario", "tester-fresh-skip-two-pass", + "--target-outcome", "tester_order_created", + "--target-outcome", "tester_fresh_order_skip", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[], options: { env?: NodeJS.ProcessEnv }) => { + spawned.push({ args: commandArgs, env: options.env }); + if (isPreflightCommand(commandArgs)) { + return commandArgs.includes("tester-pass-1-1") + ? fakePreflightChild({ ckbAvailable: "2853.99897309", ickbAvailable: "250838.31219989" }) + : fakeSuccessfulPreflightChild(); + } + testerRuns += 1; + return fakeChild(JSON.stringify(testerRuns === 1 + ? { + startTime: "now", + actions: { + requestedTesterScenario: "auto", + testerScenario: "bounded-ickb-to-ckb-limit-order", + newOrder: { giveIckb: "20", takeCkb: "18", fee: "0.2" }, + cancelledOrders: 0, + }, + txHash: txHash("78"), + ElapsedSeconds: 1, + } + : { skip: { reason: "fresh-matchable-order", txHash: txHash("78") } })); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: () => Promise.resolve(), + writeFile: () => Promise.resolve(), + }); + + const testerSpawns = spawned.filter((item) => item.args[0] === "apps/tester/dist/index.js"); + expect(exitCode).toBe(0); + expect(testerSpawns).toHaveLength(2); + expect(testerSpawns[0]?.env).toMatchObject({ TESTER_SCENARIO: "auto" }); + expect(testerSpawns[1]?.env).toMatchObject({ TESTER_SCENARIO: "auto" }); + }); + + it("treats tester first-pass fresh-skip reserve misses as classified no-progress", async () => { + const spawned: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []; + const writes = new Map(); + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/two-pass-bounded-reserve-test", + "--scenario", "tester-fresh-skip-two-pass", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[], options: { env?: NodeJS.ProcessEnv }) => { + spawned.push({ args: commandArgs, env: options.env }); + return isPreflightCommand(commandArgs) + ? fakePreflightChild({ ckbAvailable: "2100", ickbAvailable: "250838.31219989" }) + : fakeChild(JSON.stringify({ skip: { reason: "post-tx-ckb-reserve" } })); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: () => Promise.resolve(), + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + const testerSpawns = spawned.filter((item) => item.args[0] === "apps/tester/dist/index.js"); + expect(exitCode).toBe(0); + expect(testerSpawns[0]?.env).toMatchObject({ TESTER_SCENARIO: "auto" }); + expect([...writes.keys()].some((path) => path.endsWith("incident.json"))).toBe(false); + const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/two-pass-bounded-reserve-test/summary.json"); + expect(summary).toMatchObject({ stopped: "max_cycles", skipReasons: ["post-tx-ckb-reserve", "post-tx-ckb-reserve"] }); + expect(recordAt(summary["aggregateCounts"], "summary aggregate counts")).toEqual({ tester_reserve_skip: 2 }); + }); + + it("uses auto first-pass fresh-skip stimulus when plain CKB is very low", async () => { + const spawned: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/two-pass-low-reserve-test", + "--scenario", "tester-fresh-skip-two-pass", + "--target-outcome", "tester_order_created", + "--target-outcome", "tester_fresh_order_skip", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[], options: { env?: NodeJS.ProcessEnv }) => { + spawned.push({ args: commandArgs, env: options.env }); + return isPreflightCommand(commandArgs) + ? fakePreflightChild({ ckbAvailable: "1999.99999999", ickbAvailable: "250838.31219989" }) + : fakeChild(JSON.stringify({ skip: { reason: "post-tx-ckb-reserve" } })); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: () => Promise.resolve(), + writeFile: () => Promise.resolve(), + }); + + const tester = spawned.find((item) => item.args[0] === "apps/tester/dist/index.js"); + expect(tester?.env).toMatchObject({ TESTER_SCENARIO: "auto" }); + }); + + it("uses auto first-pass fresh-skip stimulus when plain CKB is high", async () => { + const spawned: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []; + let testerRuns = 0; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/two-pass-high-ckb-test", + "--scenario", "tester-fresh-skip-two-pass", + "--target-outcome", "tester_order_created", + "--target-outcome", "tester_fresh_order_skip", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[], options: { env?: NodeJS.ProcessEnv }) => { + spawned.push({ args: commandArgs, env: options.env }); + if (isPreflightCommand(commandArgs)) { + return commandArgs.includes("tester-pass-1-1") + ? fakePreflightChild({ ckbAvailable: "3000", ickbAvailable: "250838.31219989" }) + : fakeSuccessfulPreflightChild(); + } + testerRuns += 1; + return fakeChild(JSON.stringify(testerRuns === 1 + ? { + startTime: "now", + actions: { + requestedTesterScenario: "auto", + testerScenario: "bounded-ickb-to-ckb-limit-order", + newOrder: { giveIckb: "20", takeCkb: "18", fee: "0.2" }, + cancelledOrders: 0, + }, + txHash: txHash("80"), + ElapsedSeconds: 1, + } + : { skip: { reason: "fresh-matchable-order", txHash: txHash("80") } })); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: () => Promise.resolve(), + writeFile: () => Promise.resolve(), + }); + + const testerSpawns = spawned.filter((item) => item.args[0] === "apps/tester/dist/index.js"); + expect(exitCode).toBe(0); + expect(testerSpawns[0]?.env).toMatchObject({ TESTER_SCENARIO: "auto" }); + expect(testerSpawns[1]?.env).toMatchObject({ TESTER_SCENARIO: "auto" }); + }); + + it("preserves explicit tester scenario during fresh-skip pass selection", async () => { + const spawned: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/two-pass-explicit-test", + "--scenario", "tester-fresh-skip-two-pass", + "--tester-scenario", "multi-order-limit-orders", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[], options: { env?: NodeJS.ProcessEnv }) => { + spawned.push({ args: commandArgs, env: options.env }); + return isPreflightCommand(commandArgs) + ? fakePreflightChild({ ckbAvailable: "2853.99897309", ickbAvailable: "250838.31219989" }) + : fakeChild(JSON.stringify({ + startTime: "now", + actions: { + requestedTesterScenario: "auto", + testerScenario: "bounded-ickb-to-ckb-limit-order", + newOrder: { giveIckb: "20", takeCkb: "18", fee: "0.2" }, + cancelledOrders: 0, + }, + txHash: txHash("79"), + ElapsedSeconds: 1, + })); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: () => Promise.resolve(), + writeFile: () => Promise.resolve(), + }); + + const tester = spawned.find((item) => item.args[0] === "apps/tester/dist/index.js"); + expect(tester?.env).toMatchObject({ TESTER_SCENARIO: "multi-order-limit-orders" }); + }); + it("refuses live config paths through symlinked parents", async () => { const args = parseArgs([ "--out-dir", "logs/live-supervisor/config-symlink-test", @@ -585,6 +833,28 @@ describe("classification", () => { }); }); + it("rejects committed tester evidence without action evidence", () => { + expect(classifyActorResult("tester", commandResult("tester", JSON.stringify({ + startTime: "now", + txHash: txHash("25"), + ElapsedSeconds: 1, + })))).toMatchObject({ + outcome: "malformed_evidence", + terminal: true, + reason: "tester committed transaction evidence did not include action evidence", + }); + expect(classifyActorResult("tester", commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { cancelledOrders: 0 }, + txHash: txHash("26"), + ElapsedSeconds: 1, + })))).toMatchObject({ + outcome: "malformed_evidence", + terminal: true, + reason: "tester committed transaction evidence did not include action evidence", + }); + }); + it("classifies tester SDK order conversions as conversion coverage", () => { const result = commandResult("tester", JSON.stringify({ startTime: "now", @@ -606,7 +876,8 @@ describe("classification", () => { actions: { testerScenario: "ickb-to-ckb-limit-order", newOrder: { giveIckb: "10", takeCkb: "9", fee: "0.1" }, - cancelledOrders: 0, + collectedOrders: 2, + cancelledOrders: 1, }, txHash: txHash("15"), ElapsedSeconds: 1, @@ -615,6 +886,38 @@ describe("classification", () => { expect(classifyActorResult("tester", result, { scenario: "ickb-to-ckb-limit-order" })).toMatchObject({ outcome: "tester_order_created", terminal: false, + testerOrder: { + testerScenario: "ickb-to-ckb-limit-order", + orderCount: 1, + collectedOrders: 2, + cancelledOrders: 1, + orders: [{ direction: "ickb-to-ckb", giveIckb: "10", takeCkb: "9", fee: "0.1", dust: false }], + }, + }); + }); + + it("captures dust tester order evidence", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + requestedTesterScenario: "auto", + testerScenario: "dust-ckb-conversion", + newOrder: { giveCkb: "0.00000001", takeIckb: "0.00000001", fee: "0", feeNumerator: "1", feeBase: "100000" }, + cancelledOrders: 1, + }, + txHash: txHash("17"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result)).toMatchObject({ + outcome: "tester_order_created", + testerOrder: { + requestedTesterScenario: "auto", + testerScenario: "dust-ckb-conversion", + orderCount: 1, + cancelledOrders: 1, + orders: [{ direction: "ckb-to-ickb", giveCkb: "0.00000001", takeIckb: "0.00000001", fee: "0", feeNumerator: "1", feeBase: "100000", dust: true }], + }, }); }); @@ -918,6 +1221,24 @@ describe("classification", () => { }); }); + it("accepts bounded iCKB-to-CKB tester scenario evidence", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + testerScenario: "bounded-ickb-to-ckb-limit-order", + newOrder: { giveIckb: "10", takeCkb: "9", fee: "0.1" }, + cancelledOrders: 0, + }, + txHash: txHash("18"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result, { scenario: "bounded-ickb-to-ckb-limit-order" })).toMatchObject({ + outcome: "tester_order_created", + terminal: false, + }); + }); + it("requires conversion evidence for explicit SDK conversion scenarios", () => { const result = commandResult("tester", JSON.stringify({ startTime: "now", @@ -941,6 +1262,13 @@ describe("classification", () => { expect(classifyActorResult("tester", commandResult("tester", JSON.stringify({ skip: { reason: "fresh-matchable-order", txHash: txHash("22") }, }))).outcome).toBe("tester_fresh_order_skip"); + expect(classifyActorResult("tester", commandResult("tester", JSON.stringify({ + skip: { reason: "matchable-order-transaction-missing", txHash: txHash("23") }, + })))).toMatchObject({ + outcome: "tester_fresh_order_skip", + terminal: false, + skipReason: "matchable-order-transaction-missing", + }); expect(classifyActorResult("tester", commandResult("tester", JSON.stringify({ skip: { reason: "sampled-amount-too-small" }, }))).outcome).toBe("tester_sampled_too_small_skip"); @@ -993,15 +1321,75 @@ describe("classification", () => { }); }); - it("keeps match diagnostics tied to the matching state-read iteration", () => { + it("classifies matched withdrawal requests as withdrawal coverage", () => { const stdout = [ - botEvent("bot.state.read", { - iterationId: 1, - orders: { marketCount: 4, userCount: 0, receiptCount: 1 }, - poolDeposits: { readyCount: 2, nearReadyCount: 1, futureCount: 3 }, + botEvent("bot.transaction.built", { + actions: { collectedOrders: 0, completedDeposits: 0, matchedOrders: 1, deposits: 0, withdrawalRequests: 1, withdrawals: 0 }, }), + botEvent("bot.transaction.committed", { txHash: txHash("39"), status: "committed" }), + ].map(JSON.stringify).join("\n"); + + expect(classifyActorResult("bot", commandResult("bot", stdout))).toMatchObject({ + outcome: "bot_withdrawal_request_committed", + actions: { matchedOrders: 1, deposits: 0, withdrawalRequests: 1 }, + txHashes: [txHash("39")], + }); + }); + + it("classifies matched receipt completions as receipt coverage", () => { + const stdout = [ botEvent("bot.transaction.built", { - iterationId: 1, + actions: { collectedOrders: 0, completedDeposits: 1, matchedOrders: 1, deposits: 0, withdrawalRequests: 0, withdrawals: 0 }, + }), + botEvent("bot.transaction.committed", { txHash: txHash("40"), status: "committed" }), + ].map(JSON.stringify).join("\n"); + + expect(classifyActorResult("bot", commandResult("bot", stdout))).toMatchObject({ + outcome: "bot_receipt_completion_committed", + actions: { completedDeposits: 1, matchedOrders: 1, deposits: 0 }, + txHashes: [txHash("40")], + }); + }); + + it("classifies matched withdrawal completions as withdrawal completion coverage", () => { + const stdout = [ + botEvent("bot.transaction.built", { + actions: { collectedOrders: 0, completedDeposits: 0, matchedOrders: 1, deposits: 0, withdrawalRequests: 0, withdrawals: 1 }, + }), + botEvent("bot.transaction.committed", { txHash: txHash("41"), status: "committed" }), + ].map(JSON.stringify).join("\n"); + + expect(classifyActorResult("bot", commandResult("bot", stdout))).toMatchObject({ + outcome: "bot_withdrawal_completion_committed", + actions: { matchedOrders: 1, deposits: 0, withdrawals: 1 }, + txHashes: [txHash("41")], + }); + }); + + it("classifies deposit-only commits as deposit coverage", () => { + const stdout = [ + botEvent("bot.transaction.built", { + actions: { collectedOrders: 0, completedDeposits: 0, matchedOrders: 0, deposits: 1, withdrawalRequests: 0, withdrawals: 0 }, + }), + botEvent("bot.transaction.committed", { txHash: txHash("42"), status: "committed" }), + ].map(JSON.stringify).join("\n"); + + expect(classifyActorResult("bot", commandResult("bot", stdout))).toMatchObject({ + outcome: "bot_deposit_only_committed", + actions: { matchedOrders: 0, deposits: 1 }, + txHashes: [txHash("42")], + }); + }); + + it("keeps match diagnostics tied to the matching state-read iteration", () => { + const stdout = [ + botEvent("bot.state.read", { + iterationId: 1, + orders: { marketCount: 4, userCount: 0, receiptCount: 1 }, + poolDeposits: { readyCount: 2, nearReadyCount: 1, futureCount: 3 }, + }), + botEvent("bot.transaction.built", { + iterationId: 1, actions: { collectedOrders: 0, completedDeposits: 0, matchedOrders: 1, deposits: 1, withdrawalRequests: 0, withdrawals: 0 }, decision: { match: { @@ -1041,6 +1429,165 @@ describe("classification", () => { nearReadyPoolDepositCount: 0, futurePoolDepositCount: 0, }); + expect(classification).toMatchObject({ + outcome: "bot_retryable_error", + terminal: false, + reason: "bot reported retryable iteration failure", + }); + }); + + it("classifies bot committed actions from the matching iteration", () => { + const stdout = [ + botEvent("bot.iteration.failed", { + iterationId: 0, + retryable: false, + terminal: true, + error: { name: "Error", message: "L1 state scan crossed chain tip; retry with a fresh state" }, + }), + botEvent("bot.transaction.built", { + iterationId: 1, + actions: { collectedOrders: 0, completedDeposits: 0, matchedOrders: 1, deposits: 0, withdrawalRequests: 0, withdrawals: 0 }, + }), + botEvent("bot.transaction.built", { + iterationId: 2, + actions: { collectedOrders: 0, completedDeposits: 0, matchedOrders: 0, deposits: 1, withdrawalRequests: 0, withdrawals: 0 }, + }), + botEvent("bot.transaction.committed", { iterationId: 1, txHash: txHash("37"), status: "committed" }), + ].map(JSON.stringify).join("\n"); + + expect(classifyActorResult("bot", commandResult("bot", stdout))).toMatchObject({ + outcome: "bot_match_committed", + actions: { matchedOrders: 1, deposits: 0 }, + }); + }); + + it("classifies later bot commits over older transaction failures", () => { + const stdout = [ + botEvent("bot.transaction.built", { + iterationId: 1, + actions: { collectedOrders: 0, completedDeposits: 0, matchedOrders: 1, deposits: 0, withdrawalRequests: 0, withdrawals: 0 }, + }), + botEvent("bot.transaction.failed", { + iterationId: 1, + outcome: "post_broadcast_unresolved", + txHash: txHash("43"), + }), + botEvent("bot.transaction.committed", { iterationId: 1, txHash: txHash("44"), status: "committed" }), + ].map(JSON.stringify).join("\n"); + + expect(classifyActorResult("bot", commandResult("bot", stdout))).toMatchObject({ + outcome: "bot_match_committed", + terminal: false, + actions: { matchedOrders: 1, deposits: 0 }, + }); + }); + + it("classifies later bot transaction failures over older commits", () => { + const stdout = [ + botEvent("bot.transaction.built", { + iterationId: 1, + actions: { collectedOrders: 0, completedDeposits: 0, matchedOrders: 1, deposits: 0, withdrawalRequests: 0, withdrawals: 0 }, + }), + botEvent("bot.transaction.committed", { iterationId: 1, txHash: txHash("45"), status: "committed" }), + botEvent("bot.transaction.failed", { + iterationId: 1, + outcome: "terminal_rejection", + txHash: txHash("46"), + }), + ].map(JSON.stringify).join("\n"); + + expect(classifyActorResult("bot", commandResult("bot", stdout))).toMatchObject({ + outcome: "terminal_chain_rejection", + terminal: true, + }); + }); + + it("rejects bot post-broadcast failures without a valid tx hash", () => { + const stdout = JSON.stringify(botEvent("bot.transaction.failed", { + outcome: "post_broadcast_unresolved", + txHash: "not-a-tx-hash", + })); + + expect(classifyActorResult("bot", commandResult("bot", stdout))).toMatchObject({ + outcome: "malformed_evidence", + terminal: true, + reason: "bot post-broadcast transaction failure evidence did not include a valid tx hash", + }); + }); + + it("keeps pre-broadcast bot failures classified without tx hash evidence", () => { + const stdout = JSON.stringify(botEvent("bot.transaction.failed", { + outcome: "validation_failed", + phase: "pre_broadcast", + })); + + expect(classifyActorResult("bot", commandResult("bot", stdout))).toMatchObject({ + outcome: "unknown", + terminal: true, + reason: "bot pre-broadcast transaction failure", + }); + }); + + it("classifies bot skips after earlier terminal iteration failures", () => { + const stdout = [ + botEvent("bot.iteration.failed", { + iterationId: 1, + retryable: false, + terminal: true, + error: { name: "Error", message: "L1 state scan crossed chain tip; retry with a fresh state" }, + }), + botEvent("bot.decision.skipped", { + iterationId: 2, + reason: "no_actions", + actions: emptyActions(), + }), + ].map(JSON.stringify).join("\n"); + + expect(classifyActorResult("bot", commandResult("bot", stdout))).toMatchObject({ + outcome: "bot_no_action_skip", + terminal: false, + skipReason: "no_actions", + }); + }); + + it("classifies bot skips after earlier retryable pre-broadcast failures", () => { + const stdout = [ + botEvent("bot.transaction.failed", { + iterationId: 1, + phase: "pre_broadcast", + outcome: "pre_broadcast_failed", + retryable: true, + terminal: false, + error: { name: "TypeError", message: "fetch failed" }, + }), + botEvent("bot.decision.skipped", { + iterationId: 2, + reason: "no_actions", + actions: emptyActions(), + }), + ].map(JSON.stringify).join("\n"); + + expect(classifyActorResult("bot", commandResult("bot", stdout))).toMatchObject({ + outcome: "bot_no_action_skip", + terminal: false, + skipReason: "no_actions", + }); + }); + + it("rejects committed bot evidence without matching built action evidence", () => { + const stdout = [ + botEvent("bot.transaction.built", { + iterationId: 1, + actions: { collectedOrders: 0, completedDeposits: 0, matchedOrders: 1, deposits: 0, withdrawalRequests: 0, withdrawals: 0 }, + }), + botEvent("bot.transaction.committed", { iterationId: 2, txHash: txHash("38"), status: "committed" }), + ].map(JSON.stringify).join("\n"); + + expect(classifyActorResult("bot", commandResult("bot", stdout))).toMatchObject({ + outcome: "malformed_evidence", + terminal: true, + reason: "bot committed transaction evidence did not include matching built action evidence", + }); }); it("rejects committed bot evidence without a valid tx hash", () => { @@ -1108,6 +1655,42 @@ describe("classification", () => { }); }); + it("treats nonzero bot exits as terminal even with retryable iteration evidence", () => { + const result = { + ...commandResult("bot", JSON.stringify(botEvent("bot.iteration.failed", { + retryable: true, + terminal: false, + error: { name: "TypeError", message: "fetch failed" }, + }))), + status: 1, + }; + + expect(classifyActorResult("bot", result)).toMatchObject({ + outcome: "nonzero_exit", + terminal: true, + }); + }); + + it("keeps terminal bot retry-budget exhaustion classified by bot evidence despite exit code 2", () => { + const result = { + ...commandResult("bot", JSON.stringify(botEvent("bot.iteration.failed", { + retryable: true, + terminal: true, + retryableAttempts: 3, + maxRetryableAttempts: 3, + retryBudgetExhausted: true, + error: { name: "TypeError", message: "fetch failed" }, + }))), + status: 2, + }; + + expect(classifyActorResult("bot", result)).toMatchObject({ + outcome: "bot_retryable_error", + terminal: true, + reason: "bot reported terminal retryable iteration failure", + }); + }); + it("reports spawn errors before generic actor exit classification", () => { expect(classifyActorResult("preflight", { ...commandResult("preflight", ""), spawnError: "ENOENT", status: null })).toMatchObject({ outcome: "nonzero_exit", @@ -1135,6 +1718,90 @@ describe("classification", () => { message: "Transaction confirmation timed out", txHash: txHash("aa"), status: "sent", + isTimeout: true, + }, + })), + status: 2, + }; + + expect(classifyActorResult("tester", result)).toMatchObject({ + outcome: "confirmation_timeout", + terminal: true, + }); + }); + + it("classifies serialized tester post-broadcast unresolved failures", () => { + const result = { + ...commandResult("tester", JSON.stringify({ + txHash: txHash("ab"), + error: { + name: "TransactionConfirmationError", + message: "Transaction confirmation timed out", + txHash: txHash("ab"), + status: "sent", + isTimeout: true, + cause: { name: "TypeError", message: "fetch failed" }, + }, + })), + status: 2, + }; + + expect(classifyActorResult("tester", result)).toMatchObject({ + outcome: "post_broadcast_unresolved", + terminal: true, + reason: "tester tx remained unresolved after broadcast", + }); + }); + + it("classifies serialized tester terminal chain rejections", () => { + const result = { + ...commandResult("tester", JSON.stringify({ + txHash: txHash("ac"), + error: { + name: "TransactionConfirmationError", + message: "Transaction reached rejected status", + txHash: txHash("ac"), + status: "rejected", + isTimeout: false, + }, + })), + status: 1, + }; + + expect(classifyActorResult("tester", result)).toMatchObject({ + outcome: "terminal_chain_rejection", + terminal: true, + reason: "tester tx reached terminal chain rejection", + }); + }); + + it("rejects tester transaction failures without valid tx hash evidence", () => { + const result = { + ...commandResult("tester", JSON.stringify({ + error: { + name: "TransactionConfirmationError", + message: "Transaction confirmation timed out", + isTimeout: true, + }, + })), + status: 2, + }; + + expect(classifyActorResult("tester", result)).toMatchObject({ + outcome: "malformed_evidence", + terminal: true, + reason: "tester transaction failure evidence did not include a valid tx hash", + }); + }); + + it("extracts nested tester transaction failure tx hash evidence", () => { + const result = { + ...commandResult("tester", JSON.stringify({ + error: { + name: "TransactionConfirmationError", + message: "Transaction confirmation timed out", + txHash: txHash("ad"), + isTimeout: true, }, })), status: 2, @@ -1143,6 +1810,7 @@ describe("classification", () => { expect(classifyActorResult("tester", result)).toMatchObject({ outcome: "confirmation_timeout", terminal: true, + txHashes: [txHash("ad")], }); }); @@ -1163,137 +1831,456 @@ describe("classification", () => { }); }); - it("safety classifications override ordinary exits", () => { + it("safety classifications preserve ordinary command precedence", () => { expect(classifyActorResult("bot", commandResult("bot", "{not-json}"))).toMatchObject({ outcome: "malformed_evidence", terminal: true, }); - expect(classifyActorResult("bot", commandResult("bot", JSON.stringify({ privateKey: "0xsecret" })))).toMatchObject({ - outcome: "secret_leak_sentinel", - terminal: true, - }); expect(classifyActorResult("bot", { ...commandResult("bot", ""), timedOut: true })).toMatchObject({ outcome: "command_timeout", terminal: true, }); + expect(classifyActorResult("bot", commandResult("bot", [ + JSON.stringify(botEvent("bot.decision.skipped", { reason: "no_actions", actions: emptyActions() })), + JSON.stringify({ witnesses: ["0xsignature"], inputs: [] }), + ].join("\n")))).toMatchObject({ + outcome: "bot_no_action_skip", + terminal: false, + }); }); - it("classifies terminal preflight safety failures before launch", () => { + it("classifies terminal preflight command failures before launch", () => { expect(classifyActorResult("preflight", { ...commandResult("preflight", ""), timedOut: true })).toMatchObject({ outcome: "command_timeout", terminal: true, }); + }); + + it("requires preflight configs to bound actors to one iteration", () => { + expect(classifyActorResult("preflight", commandResult("preflight", JSON.stringify({ + chain: "testnet", + bounded: false, + })))).toMatchObject({ + outcome: "malformed_evidence", + terminal: true, + reason: "preflight config is not bounded to one iteration", + }); expect(classifyActorResult("preflight", commandResult("preflight", JSON.stringify({ - privateKey: "0xsecret", + chain: "testnet", + bounded: true, + maxIterations: 2, })))).toMatchObject({ - outcome: "secret_leak_sentinel", + outcome: "malformed_evidence", terminal: true, + reason: "preflight config is not bounded to one iteration", }); }); - it("sanitizes transaction-shaped preflight errors before classification reaches artifacts", () => { + it("fails closed when captured command output is truncated", () => { + expect(classifyActorResult("bot", { + ...commandResult("bot", JSON.stringify(botEvent("bot.decision.skipped", { + reason: "no_actions", + actions: emptyActions(), + }))), + stdoutTruncated: true, + })).toMatchObject({ + outcome: "malformed_evidence", + terminal: true, + reason: "stdout evidence exceeded supervisor capture limit", + evidence: { stdoutTruncated: true }, + }); + expect(classifyActorResult("preflight", { + ...commandResult("preflight", JSON.stringify({ chain: "testnet", bounded: true, maxIterations: 1 })), + stderrTruncated: true, + })).toMatchObject({ + outcome: "malformed_evidence", + terminal: true, + reason: "stderr evidence exceeded supervisor capture limit", + evidence: { stderrTruncated: true }, + }); + }); + + it("preserves transaction-shaped preflight stderr for artifact capture", () => { const classification = classifyActorResult("preflight", { ...commandResult("preflight", "{}"), status: 1, stderr: JSON.stringify({ witnesses: ["0xsignature"], inputs: [] }), }); - expect(classification.reason).toBe("\n"); + expect(classification).toMatchObject({ + outcome: "nonzero_exit", + terminal: true, + reason: JSON.stringify({ witnesses: ["0xsignature"], inputs: [] }), + }); }); - it("withholds secret-shaped raw artifacts", () => { - expect(safeArtifactText(JSON.stringify({ privateKey: "0xsecret" }))).toBe( - "\n", - ); - expect(safeArtifactText(`PRIVATE_KEY=0x${"11".repeat(32)}`)).toBe( - "\n", - ); - expect(safeArtifactText("SEED_PHRASE=alpha beta gamma")).toBe( - "\n", - ); - expect(safeArtifactText("RPC_URL=https://user:pass@testnet.example/path?token=secret")).toBe( - "\n", - ); - expect(safeArtifactText(JSON.stringify({ witnesses: ["0xsignature"], inputs: [] }))).toBe( - "\n", - ); - expect(safeArtifactText(JSON.stringify({ - transactionShape: { inputs: 1, outputs: 2, cellDeps: 3, headerDeps: 4, witnesses: 5 }, - }))).toBe( - JSON.stringify({ transactionShape: { inputs: 1, outputs: 2, cellDeps: 3, headerDeps: 4, witnesses: 5 } }), + it("preserves snake_case CKB transaction fields for artifact capture", () => { + const classification = classifyActorResult("preflight", { + ...commandResult("preflight", "{}"), + status: 1, + stderr: JSON.stringify({ cell_deps: [], header_deps: [], outputs_data: ["0x"] }), + }); + + expect(classification).toMatchObject({ + outcome: "nonzero_exit", + terminal: true, + reason: JSON.stringify({ cell_deps: [], header_deps: [], outputs_data: ["0x"] }), + }); + }); + + it("classifies retryable preflight transport failures separately", () => { + expect(classifyActorResult("preflight", { + ...commandResult("preflight", ""), + status: 1, + stderr: "Live preflight retryable failure: fetch failed\n", + })).toMatchObject({ + outcome: "preflight_retryable_error", + terminal: true, + }); + }); + + it("classifies preserved wrong-chain preflight evidence", () => { + expect(classifyActorResult("preflight", { + ...commandResult("preflight", ""), + status: 1, + stderr: "Live preflight failed: Invalid testnet RPC chain identity: genesis hash expected 0x1 observed 0x2\n", + })).toMatchObject({ + outcome: "wrong_chain", + terminal: true, + }); + }); + + it("caps captured command output", () => { + const capture = createBoundedOutputCapture(); + appendBoundedOutput(capture, Buffer.from("abcdef"), 4); + appendBoundedOutput(capture, Buffer.from("gh"), 4); + + expect(boundedOutputText(capture)).toBe("abcd\n"); + }); + + it("kills timed-out actor commands after the grace period", async () => { + vi.useFakeTimers(); + try { + const writes = new Map(); + const kills: Array<{ pid: number; signal: NodeJS.Signals }> = []; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/timeout-kill-test", + "--scenario", "bot-only", + "--target-outcome", "bot_match_committed", + "--max-cycles", "1", + "--command-timeout-seconds", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/timeout-kill-test", + "config/bot-testnet.json", + "config/tester-testnet.json", + ])) }); + const child = fakeHangingChild(); + + const run = supervise(args, plan, { + skipBuiltRuntimeCheck: true, + commandKillGraceMs: 10, + killProcess: (pid, signal) => { + kills.push({ pid, signal }); + if (signal === "SIGKILL") { + queueMicrotask(() => child.emit("close", null, "SIGKILL")); + } + }, + spawnCommand: ((_command: string, commandArgs: string[]) => isPreflightCommand(commandArgs) ? fakeSuccessfulPreflightChild() : child) as never, + spawnSyncCommand: ignoredChecker(true) as never, + lstat: missingStat, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + realpath: (path) => Promise.resolve(pathToString(path)), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + await vi.advanceTimersByTimeAsync(1010); + + await expect(run).resolves.toBe(2); + expect(kills).toEqual([ + { pid: -1234, signal: "SIGTERM" }, + { pid: -1234, signal: "SIGKILL" }, + ]); + expect(writes.get("/repo/logs/live-supervisor/timeout-kill-test/cycle-0001-incident.json")).toContain( + "command_timeout", + ); + } finally { + vi.useRealTimers(); + } + }); + + it("bounds in-flight actor commands by the remaining wall-clock budget", async () => { + vi.useFakeTimers(); + try { + const writes = new Map(); + const kills: Array<{ pid: number; signal: NodeJS.Signals }> = []; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/wall-clock-timeout-test", + "--scenario", "bot-only", + "--target-outcome", "bot_match_committed", + "--max-cycles", "1", + "--max-wall-clock-seconds", "1", + "--command-timeout-seconds", "900", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/wall-clock-timeout-test", + "config/bot-testnet.json", + "config/tester-testnet.json", + ])) }); + const child = fakeHangingChild(); + + const run = supervise(args, plan, { + skipBuiltRuntimeCheck: true, + commandKillGraceMs: 10, + killProcess: (pid, signal) => { + kills.push({ pid, signal }); + if (signal === "SIGKILL") { + queueMicrotask(() => child.emit("close", null, "SIGKILL")); + } + }, + spawnCommand: ((_command: string, commandArgs: string[]) => isPreflightCommand(commandArgs) ? fakeSuccessfulPreflightChild() : child) as never, + spawnSyncCommand: ignoredChecker(true) as never, + lstat: missingStat, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + realpath: (path) => Promise.resolve(pathToString(path)), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + await vi.advanceTimersByTimeAsync(1010); + + await expect(run).resolves.toBe(2); + expect(kills).toEqual([ + { pid: -1234, signal: "SIGTERM" }, + { pid: -1234, signal: "SIGKILL" }, + ]); + const incident = jsonArtifact(writes, "/repo/logs/live-supervisor/wall-clock-timeout-test/cycle-0001-incident.json"); + expect(recordAt(incident["classification"], "incident classification")).toMatchObject({ + outcome: "command_timeout", + }); + } finally { + vi.useRealTimers(); + } + }); + + it("does not spawn actor commands when wall-clock expires at the command boundary", async () => { + const writes = new Map(); + const spawned: string[][] = []; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/wall-clock-boundary-test", + "--scenario", "bot-only", + "--target-outcome", "bot_match_committed", + "--max-cycles", "1", + "--max-wall-clock-seconds", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/wall-clock-boundary-test", + "config/bot-testnet.json", + "config/tester-testnet.json", + ])) }); + const clock = [0, 999, 1000, 1000]; + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + now: () => clock.shift() ?? 1000, + spawnCommand: ((_command: string, commandArgs: string[]) => { + spawned.push(commandArgs); + return fakeChild("should not run"); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + lstat: missingStat, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + realpath: (path) => Promise.resolve(pathToString(path)), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(2); + expect(spawned).toEqual([]); + const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/wall-clock-boundary-test/summary.json"); + expect(summary).toMatchObject({ stopped: "unmet_coverage_goal" }); + }); + + it("retries retryable preflight transport failures once before actor execution", async () => { + const writes = new Map(); + const spawned: string[][] = []; + let preflightRuns = 0; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/preflight-retry-test", + "--scenario", "bot-only", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/preflight-retry-test", + "config/bot-testnet.json", + "config/tester-testnet.json", + ])) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[]) => { + spawned.push(commandArgs); + if (isPreflightCommand(commandArgs)) { + preflightRuns += 1; + return preflightRuns === 1 + ? fakeChild("", 1, "Live preflight retryable failure: fetch failed\n") + : fakeSuccessfulPreflightChild(); + } + return fakeChild(JSON.stringify(botEvent("bot.decision.skipped", { + reason: "no_actions", + actions: emptyActions(), + }))); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + lstat: missingStat, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + realpath: (path) => Promise.resolve(pathToString(path)), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(0); + expect(spawned.filter((args) => isPreflightCommand(args))).toHaveLength(2); + expect(spawned.filter((args) => !isPreflightCommand(args))).toHaveLength(1); + expect(writes.has("/repo/logs/live-supervisor/preflight-retry-test/cycle-0001-bot-preflight-attempt-1.stdout.json")).toBe(true); + expect(writes.has("/repo/logs/live-supervisor/preflight-retry-test/cycle-0001-bot-preflight-attempt-1.stdout.ndjson")).toBe(false); + expect(writes.has("/repo/logs/live-supervisor/preflight-retry-test/cycle-0001-bot-preflight-attempt-2.stdout.json")).toBe(true); + expect(writes.has("/repo/logs/live-supervisor/preflight-retry-test/cycle-0001-bot-preflight-attempt-1.command.json")).toBe(true); + expect(writes.has("/repo/logs/live-supervisor/preflight-retry-test/cycle-0001-bot-preflight-attempt-2.command.json")).toBe(true); + const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/preflight-retry-test/summary.json"); + expect(recordAt(summary["aggregateCounts"], "summary aggregate counts")).toMatchObject({ bot_no_action_skip: 1 }); + }); + + it("writes retryable-looking preflight output as public producer artifacts", async () => { + const writes = new Map(); + const spawned: string[][] = []; + const preflightOutput = JSON.stringify({ diagnostic: "public preflight output" }); + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/preflight-unsafe-retry-test", + "--scenario", "bot-only", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/preflight-unsafe-retry-test", + "config/bot-testnet.json", + "config/tester-testnet.json", + ])) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[]) => { + spawned.push(commandArgs); + return fakeChild(preflightOutput, 1, "Live preflight retryable failure: fetch failed\n"); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + lstat: missingStat, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + realpath: (path) => Promise.resolve(pathToString(path)), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(2); + expect(spawned.filter((args) => isPreflightCommand(args))).toHaveLength(2); + expect(spawned.filter((args) => !isPreflightCommand(args))).toHaveLength(0); + const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/preflight-unsafe-retry-test/summary.json"); + expect(summary).toMatchObject({ stopped: "preflight_retryable_error" }); + expect(recordAt(summary["aggregateCounts"], "summary aggregate counts")).toMatchObject({ preflight_retryable_error: 1 }); + expect(writes.get("/repo/logs/live-supervisor/preflight-unsafe-retry-test/cycle-0001-bot-preflight-attempt-1.stdout.json")).toBe( + `${preflightOutput}\n`, ); - expect(safeArtifactText(JSON.stringify({ system: { tip: { hash: txHash("aa") } } }))).toBe( - JSON.stringify({ system: { tip: { hash: txHash("aa") } } }), + expect(writes.get("/repo/logs/live-supervisor/preflight-unsafe-retry-test/cycle-0001-bot-preflight-attempt-2.stdout.json")).toBe( + `${preflightOutput}\n`, ); - expect(safeArtifactText(JSON.stringify({ app: "bot" }))).toBe(JSON.stringify({ app: "bot" })); - }); - - it("caps captured command output", () => { - const capture = createBoundedOutputCapture(); - appendBoundedOutput(capture, Buffer.from("abcdef"), 4); - appendBoundedOutput(capture, Buffer.from("gh"), 4); - - expect(boundedOutputText(capture)).toBe("abcd\n"); }); - it("kills timed-out actor commands after the grace period", async () => { - vi.useFakeTimers(); - try { - const writes = new Map(); - const kills: Array<{ pid: number; signal: NodeJS.Signals }> = []; - const args = parseArgs([ - "--out-dir", "logs/live-supervisor/timeout-kill-test", - "--scenario", "bot-only", - "--target-outcome", "bot_match_committed", - "--max-cycles", "1", - "--command-timeout-seconds", "1", - ]); - const plan = resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ - "logs/live-supervisor/timeout-kill-test", - "config/bot-testnet.json", - "config/tester-testnet.json", - ])) }); - const child = fakeHangingChild(); - - const run = supervise(args, plan, { - skipBuiltRuntimeCheck: true, - commandKillGraceMs: 10, - killProcess: (pid, signal) => { - kills.push({ pid, signal }); - if (signal === "SIGKILL") { - queueMicrotask(() => child.emit("close", null, "SIGKILL")); - } - }, - spawnCommand: ((_command: string, commandArgs: string[]) => isPreflightCommand(commandArgs) ? fakeSuccessfulPreflightChild() : child) as never, - spawnSyncCommand: ignoredChecker(true) as never, - lstat: missingStat, - stat: missingStat, - mkdir: () => Promise.resolve(undefined), - realpath: (path) => Promise.resolve(pathToString(path)), - appendFile: (path, text) => { - const key = pathToString(path); - writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); - return Promise.resolve(); - }, - writeFile: (path, text) => { - writes.set(pathToString(path), textToString(text)); - return Promise.resolve(); - }, - }); + it("does not retry preflight after the wall-clock budget expires", async () => { + const writes = new Map(); + const spawned: string[][] = []; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/preflight-retry-wall-clock-test", + "--scenario", "bot-only", + "--target-outcome", "bot_match_committed", + "--max-cycles", "1", + "--max-wall-clock-seconds", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/preflight-retry-wall-clock-test", + "config/bot-testnet.json", + "config/tester-testnet.json", + ])) }); + const clock = [0, 0, 0, 0, 2000]; - await vi.advanceTimersByTimeAsync(1010); + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + now: () => clock.shift() ?? 2000, + spawnCommand: ((_command: string, commandArgs: string[]) => { + spawned.push(commandArgs); + return fakeChild("", 1, "Live preflight retryable failure: fetch failed\n"); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + lstat: missingStat, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + realpath: (path) => Promise.resolve(pathToString(path)), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); - await expect(run).resolves.toBe(2); - expect(kills).toEqual([ - { pid: -1234, signal: "SIGTERM" }, - { pid: -1234, signal: "SIGKILL" }, - ]); - expect(writes.get("/repo/logs/live-supervisor/timeout-kill-test/cycle-0001-incident.json")).toContain( - "command_timeout", - ); - } finally { - vi.useRealTimers(); - } + expect(exitCode).toBe(2); + expect(spawned.filter((args) => isPreflightCommand(args))).toHaveLength(1); + expect(spawned.filter((args) => !isPreflightCommand(args))).toHaveLength(0); + const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/preflight-retry-wall-clock-test/summary.json"); + expect(summary).toMatchObject({ stopped: "unmet_coverage_goal" }); + expect(writes.has("/repo/logs/live-supervisor/preflight-retry-wall-clock-test/cycle-0001-bot-preflight-attempt-1.command.json")).toBe(true); }); it("caps actor command timers to Node's maximum delay", async () => { @@ -1430,6 +2417,17 @@ describe("scenario planning", () => { }); }); + it("auto-plans fresh order skip targets through the two-pass scenario", () => { + const ledger = createCoverageLedger(["tester_fresh_order_skip"]); + const choice = chooseScenario({ scenario: "auto", targetOutcomes: ["tester_fresh_order_skip"] }, ledger); + + expect(choice).toMatchObject({ + kind: "scenario", + scenario: { name: "tester-fresh-skip-two-pass" }, + targetOutcomes: ["tester_fresh_order_skip"], + }); + }); + it("records unsupported explicit goals as full terminal classifications", async () => { const writes = new Map(); const args = parseArgs([ @@ -1456,7 +2454,7 @@ describe("scenario planning", () => { expect(exitCode).toBe(2); const incident = jsonArtifact(writes, "/repo/logs/live-supervisor/unsupported-test/cycle-0001-incident.json"); const classification = recordAt(incident["classification"], "incident classification"); - expect(classification).toMatchObject({ actor: "preflight", outcome: "unknown", terminal: true }); + expect(classification).toMatchObject({ actor: "preflight", outcome: "unsupported_scenario", terminal: true }); expect(classification["txHashes"]).toEqual([]); expect(recordAt(classification["evidence"], "incident classification evidence")).toMatchObject({ recordsAccepted: 0, @@ -1467,7 +2465,7 @@ describe("scenario planning", () => { timedOut: false, }); const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/unsupported-test/summary.json"); - expect(recordAt(summary["aggregateCounts"], "summary aggregate counts")).toMatchObject({ unknown: 1 }); + expect(recordAt(summary["aggregateCounts"], "summary aggregate counts")).toMatchObject({ unsupported_scenario: 1 }); }); it("plans remaining target outcomes after earlier targets are covered", () => { @@ -1541,6 +2539,136 @@ describe("deterministic incident handling", () => { expect(summaryArtifacts).toContain("logs/live-supervisor/unmet-coverage-test/cycle-0001-bot.command.json"); }); + it("treats unmet explicit target outcomes at max wall-clock as logical incidents", async () => { + const writes = new Map(); + const args = parseArgs([ + "--bot-config", "config/bot-testnet.json", + "--tester-config", "config/tester-testnet.json", + "--out-dir", "logs/live-supervisor/unmet-wall-clock-test", + "--scenario", "bot-only", + "--target-outcome", "bot_match_committed", + "--max-wall-clock-seconds", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + const clock = [0, 2000]; + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + now: () => clock.shift() ?? 2000, + spawnCommand: (() => { + throw new Error("actor should not start after wall-clock expiry"); + }), + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(2); + const incident = jsonArtifact(writes, "/repo/logs/live-supervisor/unmet-wall-clock-test/cycle-0001-incident.json"); + expect(incident).toMatchObject({ unmetGoals: ["bot_match_committed"] }); + expect(recordAt(incident["classification"], "incident classification")).toMatchObject({ + actor: "preflight", + outcome: "unmet_coverage_goal", + terminal: true, + reason: "bounded wall-clock budget ended before observing requested outcomes: bot_match_committed", + }); + const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/unmet-wall-clock-test/summary.json"); + expect(summary).toMatchObject({ stopped: "unmet_coverage_goal" }); + expect(recordAt(summary["aggregateCounts"], "summary aggregate counts")).toMatchObject({ unmet_coverage_goal: 1 }); + }); + + it("uses the last attempted cycle for wall-clock unmet coverage incidents", async () => { + const writes = new Map(); + const args = parseArgs([ + "--bot-config", "config/bot-testnet.json", + "--tester-config", "config/tester-testnet.json", + "--out-dir", "logs/live-supervisor/unmet-wall-clock-after-cycle-test", + "--scenario", "bot-only", + "--target-outcome", "bot_match_committed", + "--max-wall-clock-seconds", "1", + "--max-cycles", "2", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + const clock = [0, 0, 0, 0, 0, 0, 0, 2000]; + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + now: () => clock.shift() ?? 2000, + spawnCommand: ((_command: string, commandArgs: string[]) => isPreflightCommand(commandArgs) ? fakeSuccessfulPreflightChild() : fakeChild(JSON.stringify(botEvent("bot.decision.skipped", { + reason: "no_actions", + actions: emptyActions(), + })))) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(2); + const incident = jsonArtifact(writes, "/repo/logs/live-supervisor/unmet-wall-clock-after-cycle-test/cycle-0001-incident.json"); + expect(incident).toMatchObject({ cycleIndex: 1, unmetGoals: ["bot_match_committed"] }); + expect(writes.has("/repo/logs/live-supervisor/unmet-wall-clock-after-cycle-test/cycle-0002-incident.json")).toBe(false); + }); + + it("does not start another command after the wall-clock budget expires mid-cycle", async () => { + const writes = new Map(); + const spawned: string[][] = []; + const args = parseArgs([ + "--bot-config", "config/bot-testnet.json", + "--tester-config", "config/tester-testnet.json", + "--out-dir", "logs/live-supervisor/mid-cycle-wall-clock-test", + "--scenario", "standard-cycle", + "--target-outcome", "bot_match_committed", + "--max-wall-clock-seconds", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + const clock = [0, 0, 0, 0, 0, 2000]; + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + now: () => clock.shift() ?? 2000, + spawnCommand: ((_command: string, commandArgs: string[]) => { + spawned.push(commandArgs); + return fakeSuccessfulPreflightChild(); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(2); + expect(spawned).toHaveLength(1); + expect(spawned[0]).toContain("tester-1"); + const incident = jsonArtifact(writes, "/repo/logs/live-supervisor/mid-cycle-wall-clock-test/cycle-0001-incident.json"); + expect(incident).toMatchObject({ cycleIndex: 1, unmetGoals: ["bot_match_committed"] }); + }); + it("treats repeated target outcomes as one explicit contract", async () => { const writes = new Map(); const args = parseArgs([ @@ -1616,6 +2744,7 @@ describe("deterministic incident handling", () => { expect(exitCode).toBe(2); const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/target-ledger-test/summary.json"); expect(recordAt(summary["coverage"], "coverage")["goals"]).toEqual(["tester_estimated_too_small_skip"]); + expect(summary).toMatchObject({ txCreatingTxHashCount: 1, txCreatingOutcomeCount: 1 }); }); it("keeps successful preflight probes out of aggregate outcome counts", async () => { @@ -1653,6 +2782,79 @@ describe("deterministic incident handling", () => { expect(recordAt(summary["aggregateCounts"], "summary aggregate counts")).toEqual({ bot_no_action_skip: 1 }); }); + it("summarizes safe preflight balances and selected tester scenario", async () => { + const writes = new Map(); + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/preflight-state-summary-test", + "--scenario", "tester-fresh-skip-two-pass", + "--target-outcome", "tester_order_created", + "--stop-after-tx-count", "1", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/preflight-state-summary-test", + "config/bot-testnet.json", + "config/tester-testnet.json", + ])) }); + + let testerRuns = 0; + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[]) => isPreflightCommand(commandArgs) + ? fakePreflightChild({ ckbAvailable: "2853.99897309", ickbAvailable: "250838.31219989" }) + : fakeChild(JSON.stringify((testerRuns += 1) === 1 + ? { + startTime: "now", + actions: { + requestedTesterScenario: "multi-order-limit-orders", + testerScenario: "two-ickb-to-ckb-limit-orders", + newOrders: [ + { giveIckb: "10", takeCkb: "9", fee: "0.1" }, + { giveIckb: "10", takeCkb: "9", fee: "0.1" }, + ], + orderCount: 2, + cancelledOrders: 0, + }, + txHash: txHash("81"), + ElapsedSeconds: 1, + } + : { skip: { reason: "fresh-matchable-order", txHash: txHash("81") } }))) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: () => Promise.resolve(), + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(0); + const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/preflight-state-summary-test/summary.json"); + expect(summary["preflightState"]).toEqual([ + { + cycleIndex: 1, + actor: "tester", + step: "tester-pass-1", + selectedTesterScenario: "auto", + balances: { + CKB: { available: "2853.99897309" }, + ICKB: { available: "250838.31219989" }, + }, + }, + { + cycleIndex: 1, + actor: "tester", + step: "tester-pass-2", + selectedTesterScenario: "auto", + balances: { + CKB: { available: "2853.99897309" }, + ICKB: { available: "250838.31219989" }, + }, + }, + ]); + }); + it("keeps default coverage goals best-effort at max cycles", async () => { let spawnCount = 0; const writes = new Map(); @@ -1739,7 +2941,21 @@ describe("deterministic incident handling", () => { expect(exitCode).toBe(0); expect([...writes.keys()].some((path) => path.endsWith("incident.json"))).toBe(false); - expect(writes.get("/repo/logs/live-supervisor/stop-after-tx-test/summary.json")).toContain("stop_after_tx_count"); + const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/stop-after-tx-test/summary.json"); + expect(summary).toMatchObject({ + stopped: "stop_after_tx_count", + txCreatingTxHashCount: 1, + txCreatingOutcomeCount: 1, + testerOrderEvidence: [ + { + outcome: "tester_order_created", + txHashes: [txHash("44")], + orderCount: 1, + cancelledOrders: 0, + orders: [{ direction: "ckb-to-ickb", giveCkb: "10", takeIckb: "9", fee: "0.1", dust: false }], + }, + ], + }); }); it("does not count skip reference hashes toward stop-after-tx-count", async () => { @@ -1776,7 +2992,7 @@ describe("deterministic incident handling", () => { expect(exitCode).toBe(0); const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/skip-hash-stop-test/summary.json"); - expect(summary).toMatchObject({ stopped: "max_cycles" }); + expect(summary).toMatchObject({ stopped: "max_cycles", txCreatingTxHashCount: 0, txCreatingOutcomeCount: 0 }); expect(recordAt(summary["txHashesByOutcome"], "summary tx hashes")).toEqual({ tester_fresh_order_skip: [txHash("44")], }); @@ -1841,13 +3057,35 @@ function isPreflightCommand(args: string[]): boolean { } function fakeSuccessfulPreflightChild(): ReturnType { - return fakeChild(JSON.stringify({ chain: "testnet" })); + return fakeChild(JSON.stringify({ chain: "testnet", bounded: true, maxIterations: 1 })); +} + +function fakePreflightChild({ ckbAvailable, ickbAvailable }: { ckbAvailable: string; ickbAvailable: string }): ReturnType { + return fakeChild(JSON.stringify({ + chain: "testnet", + bounded: true, + maxIterations: 1, + balances: { + CKB: { available: ckbAvailable }, + ICKB: { available: ickbAvailable }, + }, + })); } function fakeChild(stdout: string, status = 0): EventEmitter & { stdout: EventEmitter; stderr: EventEmitter; kill: () => boolean; +}; +function fakeChild(stdout: string, status: number, stderr: string): EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; + kill: () => boolean; +}; +function fakeChild(stdout: string, status = 0, stderr = ""): EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; + kill: () => boolean; } { const child = new EventEmitter() as EventEmitter & { stdout: EventEmitter; @@ -1859,6 +3097,9 @@ function fakeChild(stdout: string, status = 0): EventEmitter & { child.kill = (): boolean => true; queueMicrotask(() => { child.stdout.emit("data", Buffer.from(`${stdout}\n`)); + if (stderr !== "") { + child.stderr.emit("data", Buffer.from(stderr)); + } child.emit("close", status, null); }); return child; diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index 3a95795..52b70c6 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -1,7 +1,7 @@ import { spawn, spawnSync, type SpawnSyncReturns } from "node:child_process"; import { existsSync } from "node:fs"; import { appendFile, lstat, mkdir, realpath, stat, writeFile } from "node:fs/promises"; -import { isAbsolute, join, relative, resolve } from "node:path"; +import { isAbsolute, join, parse, relative, resolve, sep } from "node:path"; import process from "node:process"; import { fileURLToPath, pathToFileURL } from "node:url"; @@ -25,6 +25,7 @@ export const OUTCOME_KINDS = [ "tester_reserve_skip", "tester_deterministic_pre_broadcast_error", "bot_no_action_skip", + "bot_retryable_error", "bot_reserve_skip", "bot_match_committed", "bot_match_plus_deposit_committed", @@ -38,17 +39,18 @@ export const OUTCOME_KINDS = [ "post_broadcast_unresolved", "wrong_chain", "malformed_evidence", - "secret_leak_sentinel", "command_timeout", + "preflight_retryable_error", "nonzero_exit", "unmet_coverage_goal", + "unsupported_scenario", "unknown", ] as const; export type OutcomeKind = typeof OUTCOME_KINDS[number]; export type Actor = "bot" | "tester"; export type ScenarioName = "auto" | "standard-cycle" | "tester-only" | "bot-only" | "tester-fresh-skip-two-pass"; -export type TesterScenario = "auto" | "random-order" | "sdk-conversion" | "extra-large-limit-order" | "multi-order-limit-orders" | "two-ckb-to-ickb-limit-orders" | "all-ckb-limit-order" | "ickb-to-ckb-limit-order" | "two-ickb-to-ckb-limit-orders" | "mixed-direction-limit-orders" | "dust-ckb-conversion" | "dust-ickb-conversion"; +export type TesterScenario = "auto" | "random-order" | "sdk-conversion" | "extra-large-limit-order" | "multi-order-limit-orders" | "two-ckb-to-ickb-limit-orders" | "all-ckb-limit-order" | "ickb-to-ckb-limit-order" | "bounded-ickb-to-ckb-limit-order" | "two-ickb-to-ckb-limit-orders" | "mixed-direction-limit-orders" | "dust-ckb-conversion" | "dust-ickb-conversion"; type TesterDirection = "ckb-to-ickb" | "ickb-to-ckb"; interface ScenarioStep { @@ -127,10 +129,13 @@ export interface Classification { exitStatus: number | null; signal: NodeJS.Signals | null; timedOut: boolean; + stdoutTruncated: boolean; + stderrTruncated: boolean; }; actions?: ActionCounts; skipReason?: string; publicState?: PublicStateAssumption; + testerOrder?: TesterOrderEvidence; } interface TesterEvidenceExpectation { @@ -159,6 +164,38 @@ export interface PublicStateAssumption { futurePoolDepositCount?: number; } +export interface TesterOrderEvidence { + requestedTesterScenario?: string; + testerScenario?: string; + orderCount: number; + collectedOrders?: number; + cancelledOrders?: number; + orders: TesterOrderSummary[]; +} + +export interface TesterOrderSummary { + direction?: TesterDirection; + giveCkb?: string; + takeIckb?: string; + giveIckb?: string; + takeCkb?: string; + fee?: string; + feeNumerator?: string; + feeBase?: string; + dust: boolean; +} + +export interface PreflightStateSummary { + cycleIndex: number; + actor: Actor; + step: string; + selectedTesterScenario?: TesterScenario; + balances?: { + CKB?: { available: string }; + ICKB?: { available: string }; + }; +} + export interface CoverageLedger { goals: OutcomeKind[]; counts: Record; @@ -267,14 +304,14 @@ const SCENARIOS: ScenarioDefinition[] = [ { name: "tester-fresh-skip-two-pass", steps: [ - { actor: "tester", label: "tester-pass-1", testerScenario: "multi-order-limit-orders" }, + { actor: "tester", label: "tester-pass-1" }, { actor: "tester", label: "tester-pass-2" }, ], targetOutcomes: [ "tester_order_created", "tester_fresh_order_skip", ], - reason: "run the same tester twice so a multi-order first pass can leave a fresh owned order for skip coverage", + reason: "run the same tester twice so a fundable first pass can leave a fresh owned order for skip coverage", }, ]; @@ -317,7 +354,7 @@ export function usage(): string { " --max-wall-clock-seconds ", " --stop-after-tx-count ", " --scenario auto|standard-cycle|tester-only|bot-only|tester-fresh-skip-two-pass", - " --tester-scenario auto|random-order|sdk-conversion|extra-large-limit-order|multi-order-limit-orders|two-ckb-to-ickb-limit-orders|all-ckb-limit-order|ickb-to-ckb-limit-order|two-ickb-to-ckb-limit-orders|mixed-direction-limit-orders|dust-ckb-conversion|dust-ickb-conversion", + " --tester-scenario auto|random-order|sdk-conversion|extra-large-limit-order|multi-order-limit-orders|two-ckb-to-ickb-limit-orders|all-ckb-limit-order|ickb-to-ckb-limit-order|bounded-ickb-to-ckb-limit-order|two-ickb-to-ckb-limit-orders|mixed-direction-limit-orders|dust-ckb-conversion|dust-ickb-conversion", " --tester-fee Default: 1", " --tester-fee-base Default: 100000", " --target-outcome Repeatable; planner prefers these first", @@ -424,16 +461,16 @@ export function parseArgs(argv: string[]): ParsedArgs { export function resolvePlan(args: ParsedArgs, rootDir = repoRoot, dependencies: Dependencies = {}): SupervisorPlan { const runId = createRunId(); - const outDir = insideRepoPath(rootDir, args.outDir ?? `logs/live-supervisor/${runId}`, "Output directory"); - const relativeOutDir = relative(rootDir, outDir); + const outDir = resolveConfiguredPath(rootDir, args.outDir ?? `logs/live-supervisor/${runId}`, "Output directory"); + const relativeOutDir = displayPath(rootDir, outDir); const botConfigPath = args.botConfigPath === undefined ? undefined : insideRepoPath(rootDir, args.botConfigPath, "Bot config path"); const testerConfigPath = args.testerConfigPath === undefined ? undefined : insideRepoPath(rootDir, args.testerConfigPath, "Tester config path"); - assertSupervisorOutputDirectory(relativeOutDir); - if (!isIgnoredPath(rootDir, relativeOutDir, dependencies)) { + assertSupervisorOutputDirectory(outDir, relativeOutDir); + if (isInside(rootDir, outDir) && !isIgnoredPath(rootDir, relativeOutDir, dependencies)) { throw new Error(`Refusing to write non-ignored supervisor output directory: ${relativeOutDir}`); } if (botConfigPath !== undefined) { @@ -515,17 +552,11 @@ export function classifyActorResult( exitStatus: result.status, signal: result.signal, timedOut: result.timedOut, + stdoutTruncated: result.stdoutTruncated, + stderrTruncated: result.stderrTruncated, }, }; - if (containsSecretLeak(result.stdout) || containsSecretLeak(result.stderr)) { - return { - ...base, - outcome: "secret_leak_sentinel", - terminal: true, - reason: "stdout or stderr contained a secret-shaped field", - }; - } if (result.timedOut) { return { ...base, @@ -542,6 +573,16 @@ export function classifyActorResult( reason: `${actor} failed to spawn: ${result.spawnError}`, }; } + if (result.stdoutTruncated || result.stderrTruncated) { + return { + ...base, + outcome: "malformed_evidence", + terminal: true, + reason: result.stdoutTruncated + ? "stdout evidence exceeded supervisor capture limit" + : "stderr evidence exceeded supervisor capture limit", + }; + } if (evidence.malformedLines.length > 0) { return { ...base, @@ -610,7 +651,11 @@ export function chooseScenario(args: Pick scenario.targetOutcomes.includes(underCovered)) .sort((left, right) => left.steps.length - right.steps.length); - const scenario = candidates.find((candidate) => !ledger.attempts.some((attempt) => attempt.scenario === candidate.name)) + const preferred = underCovered === "tester_fresh_order_skip" + ? candidates.find((candidate) => candidate.name === "tester-fresh-skip-two-pass") + : undefined; + const scenario = preferred + ?? candidates.find((candidate) => !ledger.attempts.some((attempt) => attempt.scenario === candidate.name)) ?? candidates[0]; if (scenario === undefined) { return { @@ -661,6 +706,9 @@ export async function supervise(args: ParsedArgs, plan: SupervisorPlan, dependen async function superviseOnce(args: ParsedArgs, plan: SupervisorPlan, dependencies: Dependencies = {}): Promise { const startedAt = now(dependencies); + const wallClockDeadline = args.maxWallClockSeconds === undefined + ? undefined + : startedAt + args.maxWallClockSeconds * 1000; if (!args.dryRun && dependencies.skipBuiltRuntimeCheck !== true) { assertBuiltRuntime(plan, dependencies); } @@ -668,18 +716,34 @@ async function superviseOnce(args: ParsedArgs, plan: SupervisorPlan, dependencie const ledger = createCoverageLedger(explicitCoverageGoals(args).length > 0 ? explicitCoverageGoals(args) : DEFAULT_COVERAGE_GOALS); const classifications = new Array(); const artifacts = new Array(); + const preflightState = new Array(); let txCount = 0; let latestPublicState: PublicStateAssumption | undefined; + const stopForExpiredWallClock = async (incidentCycleIndex: number): Promise => { + if (args.maxWallClockSeconds === undefined || now(dependencies) - startedAt < args.maxWallClockSeconds * 1000) { + return undefined; + } + const unmetGoals = unmetExplicitGoals(args, ledger); + if (unmetGoals.length > 0) { + const incident = await writeUnmetCoverageIncident(plan, incidentCycleIndex, unmetGoals, ledger, artifacts, dependencies, "bounded wall-clock budget"); + classifications.push(incident.classification); + await writeSummary(plan, ledger, classifications, artifacts, preflightState, latestPublicState, incident.classification.outcome, dependencies); + return STOP_EXIT_CODE; + } + await writeSummary(plan, ledger, classifications, artifacts, preflightState, latestPublicState, "max_wall_clock_seconds", dependencies); + return 0; + }; + if (args.dryRun) { const dryRunResult = await runDryRun(plan, ledger, dependencies); return dryRunResult; } for (let cycleIndex = 1; cycleIndex <= args.maxCycles; cycleIndex += 1) { - if (args.maxWallClockSeconds !== undefined && now(dependencies) - startedAt >= args.maxWallClockSeconds * 1000) { - await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, "max_wall_clock_seconds", dependencies); - return 0; + const cycleWallClockStop = await stopForExpiredWallClock(Math.max(1, cycleIndex - 1)); + if (cycleWallClockStop !== undefined) { + return cycleWallClockStop; } const choice = chooseScenario(args, ledger); @@ -689,7 +753,7 @@ async function superviseOnce(args: ParsedArgs, plan: SupervisorPlan, dependencie classifications.push(classification); const incident = unsupportedIncident(plan, cycleIndex, choice, ledger, classification); await writeJsonArtifact(plan, `cycle-${padCycle(cycleIndex)}-incident.json`, incident, artifacts, dependencies); - await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, "unsupported_scenario", dependencies); + await writeSummary(plan, ledger, classifications, artifacts, preflightState, latestPublicState, "unsupported_scenario", dependencies); return STOP_EXIT_CODE; } @@ -701,20 +765,68 @@ async function superviseOnce(args: ParsedArgs, plan: SupervisorPlan, dependencie reason: choice.reason, }, dependencies); + const preflightReports = new Map>(); for (const step of choice.scenario.steps) { - const result = await runPreflight(step.actor, plan, cycleIndex, stepLabel(step), dependencies); - artifacts.push(...await writeCommandArtifacts(plan, cycleIndex, `${stepLabel(step)}-preflight`, result, dependencies)); - const classification = classifyActorResult("preflight", result); + const wallClockStop = await stopForExpiredWallClock(cycleIndex); + if (wallClockStop !== undefined) { + return wallClockStop; + } + const firstTimeoutMs = commandTimeoutMs(plan, wallClockDeadline, dependencies); + if (firstTimeoutMs === undefined) { + const stop = await stopForExpiredWallClock(cycleIndex); + return stop ?? STOP_EXIT_CODE; + } + const firstResult = await runPreflight(step.actor, plan, cycleIndex, stepLabel(step), dependencies, firstTimeoutMs); + const firstClassification = classifyActorResult("preflight", firstResult); + const shouldRetryPreflight = firstClassification.outcome === "preflight_retryable_error"; + if (shouldRetryPreflight) { + artifacts.push(...await writeCommandArtifacts(plan, cycleIndex, `${stepLabel(step)}-preflight-attempt-1`, firstResult, dependencies)); + const retryWallClockStop = await stopForExpiredWallClock(cycleIndex); + if (retryWallClockStop !== undefined) { + return retryWallClockStop; + } + } + let result = firstResult; + if (shouldRetryPreflight) { + const retryTimeoutMs = commandTimeoutMs(plan, wallClockDeadline, dependencies); + if (retryTimeoutMs === undefined) { + const stop = await stopForExpiredWallClock(cycleIndex); + return stop ?? STOP_EXIT_CODE; + } + result = await runPreflight(step.actor, plan, cycleIndex, stepLabel(step), dependencies, retryTimeoutMs); + } + artifacts.push(...await writeCommandArtifacts( + plan, + cycleIndex, + `${stepLabel(step)}-preflight${shouldRetryPreflight ? "-attempt-2" : ""}`, + result, + dependencies, + )); + const classification = shouldRetryPreflight ? classifyActorResult("preflight", result) : firstClassification; if (classification.terminal) { classifications.push(classification); await writeIncident(plan, cycleIndex, step.actor, choice, classification, result, ledger, artifacts, dependencies); - await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, classification.outcome, dependencies); + await writeSummary(plan, ledger, classifications, artifacts, preflightState, latestPublicState, classification.outcome, dependencies); return STOP_EXIT_CODE; } + const report = parsePreflightEvidence(result.stdout).records[0]; + if (report !== undefined) { + preflightReports.set(stepLabel(step), report); + preflightState.push(preflightStateSummary(cycleIndex, step, plan, choice.targetOutcomes, report)); + } } for (const step of choice.scenario.steps) { - const result = await runActor(step, plan, choice.targetOutcomes, dependencies); + const wallClockStop = await stopForExpiredWallClock(cycleIndex); + if (wallClockStop !== undefined) { + return wallClockStop; + } + const actorTimeoutMs = commandTimeoutMs(plan, wallClockDeadline, dependencies); + if (actorTimeoutMs === undefined) { + const stop = await stopForExpiredWallClock(cycleIndex); + return stop ?? STOP_EXIT_CODE; + } + const result = await runActor(step, plan, choice.targetOutcomes, actorTimeoutMs, dependencies); artifacts.push(...await writeCommandArtifacts(plan, cycleIndex, stepLabel(step), result, dependencies)); const classification = classifyActorResult(step.actor, result, step.actor === "tester" ? testerEvidenceExpectation(plan, step) : undefined); classifications.push(classification); @@ -734,11 +846,11 @@ async function superviseOnce(args: ParsedArgs, plan: SupervisorPlan, dependencie if (classification.terminal) { const incident = await writeIncident(plan, cycleIndex, step.actor, choice, classification, result, ledger, artifacts, dependencies); - await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, incident.classification.outcome, dependencies); + await writeSummary(plan, ledger, classifications, artifacts, preflightState, latestPublicState, incident.classification.outcome, dependencies); return classification.outcome === "nonzero_exit" ? 1 : STOP_EXIT_CODE; } if (args.stopAfterTxCount !== undefined && txCount >= args.stopAfterTxCount) { - await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, "stop_after_tx_count", dependencies); + await writeSummary(plan, ledger, classifications, artifacts, preflightState, latestPublicState, "stop_after_tx_count", dependencies); return 0; } } @@ -746,13 +858,13 @@ async function superviseOnce(args: ParsedArgs, plan: SupervisorPlan, dependencie const unmetGoals = unmetExplicitGoals(args, ledger); if (unmetGoals.length > 0) { - const incident = await writeUnmetCoverageIncident(plan, args.maxCycles, unmetGoals, ledger, artifacts, dependencies); + const incident = await writeUnmetCoverageIncident(plan, args.maxCycles, unmetGoals, ledger, artifacts, dependencies, "bounded cycle budget"); classifications.push(incident.classification); - await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, incident.classification.outcome, dependencies); + await writeSummary(plan, ledger, classifications, artifacts, preflightState, latestPublicState, incident.classification.outcome, dependencies); return STOP_EXIT_CODE; } - await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, "max_cycles", dependencies); + await writeSummary(plan, ledger, classifications, artifacts, preflightState, latestPublicState, "max_cycles", dependencies); return 0; } @@ -792,7 +904,7 @@ async function runDryRun(plan: SupervisorPlan, ledger: CoverageLedger, dependenc classifications.push(classification); const incident = unsupportedIncident(plan, 1, choice, ledger, classification); await writeJsonArtifact(plan, "dry-run-incident.json", incident, artifacts, dependencies); - await writeSummary(plan, ledger, classifications, artifacts, undefined, "unsupported_scenario", dependencies); + await writeSummary(plan, ledger, classifications, artifacts, [], undefined, "unsupported_scenario", dependencies); return STOP_EXIT_CODE; } @@ -830,7 +942,7 @@ async function runDryRun(plan: SupervisorPlan, ledger: CoverageLedger, dependenc artifacts.push(...await writeCommandArtifacts(plan, 1, `dry-run-${sample.actor}`, sample.result, dependencies)); } - await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, "dry_run", dependencies); + await writeSummary(plan, ledger, classifications, artifacts, [], latestPublicState, "dry_run", dependencies); return 0; } @@ -852,14 +964,15 @@ async function prepareOutputDirectory(plan: SupervisorPlan, dependencies: Depend async function assertNoSymlinkedOutputAncestors(plan: SupervisorPlan, dependencies: Dependencies): Promise { const lstatFn = dependencies.lstat ?? lstat; - const parts = plan.relativeOutDir.split("/").filter((part) => part !== ""); - let current = plan.rootDir; + const base = isInside(plan.rootDir, plan.outDir) ? plan.rootDir : parse(plan.outDir).root; + const parts = relative(base, plan.outDir).split(sep).filter((part) => part !== ""); + let current = base; for (const part of parts) { current = join(current, part); try { const stats = await lstatFn(current); if (stats.isSymbolicLink()) { - throw new Error(`Refusing to write supervisor artifacts through symlinked path: ${relative(plan.rootDir, current)}`); + throw new Error(`Refusing to write supervisor artifacts through symlinked path: ${displayPath(plan.rootDir, current)}`); } } catch (error) { if (isNotFoundError(error)) { @@ -885,8 +998,7 @@ async function assertRealOutputDirectory(plan: SupervisorPlan, dependencies: Dep } throw error; } - const relativeOutDir = relative(realRoot, realOutDir); - if (relativeOutDir.startsWith("..") || isAbsolute(relativeOutDir)) { + if (isInside(plan.rootDir, plan.outDir) && !isInside(realRoot, realOutDir)) { throw new Error("Supervisor output directory must stay inside the repo"); } } @@ -913,24 +1025,25 @@ function assertBuiltRuntime(plan: SupervisorPlan, dependencies: Dependencies): v } } -async function runPreflight(actor: Actor, plan: SupervisorPlan, cycleIndex: number, role: string, dependencies: Dependencies): Promise { +async function runPreflight(actor: Actor, plan: SupervisorPlan, cycleIndex: number, role: string, dependencies: Dependencies, timeoutMs: number): Promise { const configPath = actor === "bot" ? plan.botConfigPath : plan.testerConfigPath; if (configPath === undefined) { throw new Error(`Missing ${actor} config path`); } await assertNoSymlinkedConfigPath(plan.rootDir, configPath, `${actor} config path`, dependencies); - return runCommand({ + const spec: Parameters[0] = { actor: "preflight", command: process.execPath, args: ["scripts/ickb-live-preflight.mjs", "--config", configPath, "--role", `${role}-${String(cycleIndex)}`], cwd: plan.rootDir, env: liveActorEnv({ INIT_CWD: plan.rootDir }), inheritEnv: false, - timeoutMs: plan.commandTimeoutSeconds * 1000, - }, dependencies); + timeoutMs, + }; + return await runCommand(spec, dependencies); } -async function runActor(step: ScenarioStep, plan: SupervisorPlan, targetOutcomes: OutcomeKind[], dependencies: Dependencies): Promise { +async function runActor(step: ScenarioStep, plan: SupervisorPlan, targetOutcomes: OutcomeKind[], timeoutMs: number, dependencies: Dependencies): Promise { const actor = step.actor; const configPath = actor === "bot" ? plan.botConfigPath : plan.testerConfigPath; if (configPath === undefined) { @@ -950,20 +1063,39 @@ async function runActor(step: ScenarioStep, plan: SupervisorPlan, targetOutcomes ...(actor === "tester" ? testerEnv(plan, targetOutcomes, step) : {}), }), inheritEnv: false, - timeoutMs: plan.commandTimeoutSeconds * 1000, + timeoutMs, }, dependencies); } function testerEnv(plan: SupervisorPlan, targetOutcomes: OutcomeKind[], step: ScenarioStep): Record { return { - TESTER_SCENARIO: testerScenarioForTargets(plan.testerScenario, plan.testerScenarioExplicit, targetOutcomes, step.testerScenario), + TESTER_SCENARIO: testerScenarioForTargets(plan.testerScenario, plan.testerScenarioExplicit, targetOutcomes, step), ...(plan.testerFeeExplicit ? { TESTER_FEE: plan.testerFee } : {}), ...(plan.testerFeeBaseExplicit ? { TESTER_FEE_BASE: plan.testerFeeBase } : {}), }; } +function commandTimeoutMs(plan: SupervisorPlan, wallClockDeadline: number | undefined, dependencies: Dependencies): number | undefined { + const configuredTimeoutMs = plan.commandTimeoutSeconds * 1000; + if (wallClockDeadline === undefined) { + return configuredTimeoutMs; + } + const remainingMs = wallClockDeadline - now(dependencies); + return remainingMs <= 0 ? undefined : Math.min(configuredTimeoutMs, remainingMs); +} + +function preflightNonzeroOutcome(stderr: string): OutcomeKind { + if (stderr.includes("Invalid") && stderr.includes("chain identity")) { + return "wrong_chain"; + } + if (stderr.includes("Live preflight retryable failure:")) { + return "preflight_retryable_error"; + } + return "nonzero_exit"; +} + function testerEvidenceExpectation(plan: SupervisorPlan, step: ScenarioStep): TesterEvidenceExpectation | undefined { - const scenario = testerScenarioForTargets(plan.testerScenario, plan.testerScenarioExplicit, [], step.testerScenario); + const scenario = testerScenarioForTargets(plan.testerScenario, plan.testerScenarioExplicit, [], step); return scenario !== "auto" ? { scenario } : undefined; } @@ -971,11 +1103,12 @@ function testerScenarioForTargets( configuredScenario: TesterScenario, testerScenarioExplicit: boolean, targetOutcomes: OutcomeKind[], - stepScenario: TesterScenario | undefined, + step: ScenarioStep, ): TesterScenario { if (testerScenarioExplicit || configuredScenario !== "auto") { return configuredScenario; } + const stepScenario = step.testerScenario; if (stepScenario !== undefined) { return stepScenario; } @@ -985,6 +1118,39 @@ function testerScenarioForTargets( return configuredScenario; } +function preflightAvailableBalanceText(preflightReport: Record | undefined, asset: "CKB" | "ICKB"): string | undefined { + const balances = optionalRecordField(preflightReport, "balances"); + const balance = optionalRecordField(balances, asset); + return stringField(balance, "available"); +} + +function preflightStateSummary( + cycleIndex: number, + step: ScenarioStep, + plan: SupervisorPlan, + targetOutcomes: OutcomeKind[], + preflightReport: Record, +): PreflightStateSummary { + const summary: PreflightStateSummary = { + cycleIndex, + actor: step.actor, + step: stepLabel(step), + }; + const ckbAvailable = preflightAvailableBalanceText(preflightReport, "CKB"); + const ickbAvailable = preflightAvailableBalanceText(preflightReport, "ICKB"); + if (ckbAvailable !== undefined || ickbAvailable !== undefined) { + summary.balances = { + ...(ckbAvailable === undefined ? {} : { CKB: { available: ckbAvailable } }), + ...(ickbAvailable === undefined ? {} : { ICKB: { available: ickbAvailable } }), + }; + } + if (step.actor === "tester") { + const selectedTesterScenario = testerScenarioForTargets(plan.testerScenario, plan.testerScenarioExplicit, targetOutcomes, step); + summary.selectedTesterScenario = selectedTesterScenario; + } + return summary; +} + function stepLabel(step: ScenarioStep): string { return step.label ?? step.actor; } @@ -1103,9 +1269,9 @@ function classifyPreflightResult( if (result.status !== 0) { return { ...base, - outcome: result.stderr.includes("Invalid") && result.stderr.includes("chain identity") ? "wrong_chain" : "nonzero_exit", + outcome: preflightNonzeroOutcome(result.stderr), terminal: true, - reason: boundedText(safeArtifactText(result.stderr), 240) || "preflight command exited nonzero", + reason: boundedText(result.stderr, 240) || "preflight command exited nonzero", }; } const report = evidence.records[0]; @@ -1117,6 +1283,14 @@ function classifyPreflightResult( reason: "preflight did not return a JSON report", }; } + if (report["bounded"] !== true || report["maxIterations"] !== 1) { + return { + ...base, + outcome: "malformed_evidence", + terminal: true, + reason: "preflight config is not bounded to one iteration", + }; + } return { ...base, outcome: "bot_no_action_skip", @@ -1132,20 +1306,30 @@ function classifyBotResult( ): Classification { const botRecords = evidence.records.filter(isBotRecord); const publicState = latestPublicState(botRecords); - const failed = lastRecordOfType(botRecords, "bot.transaction.failed"); - if (failed !== undefined) { + const lifecycle = lastRecordOfTypes(botRecords, ["bot.transaction.failed", "bot.transaction.committed"]); + if (lifecycle?.type === "bot.transaction.failed") { + const failed = lifecycle.record; const outcome = stringField(failed, "outcome"); const phase = stringField(failed, "phase"); if (outcome === "timeout_after_broadcast") { + if (!hasValidTxHash(failed)) { + return { ...base, outcome: "malformed_evidence", terminal: true, reason: "bot post-broadcast transaction failure evidence did not include a valid tx hash", publicState }; + } return { ...base, outcome: "confirmation_timeout", terminal: true, reason: "bot tx confirmation timed out", publicState }; } if (outcome === "post_broadcast_unresolved") { + if (!hasValidTxHash(failed)) { + return { ...base, outcome: "malformed_evidence", terminal: true, reason: "bot post-broadcast transaction failure evidence did not include a valid tx hash", publicState }; + } return { ...base, outcome: "post_broadcast_unresolved", terminal: true, reason: "bot tx remained unresolved after broadcast", publicState }; } if (outcome === "terminal_rejection") { + if (!hasValidTxHash(failed)) { + return { ...base, outcome: "malformed_evidence", terminal: true, reason: "bot post-broadcast transaction failure evidence did not include a valid tx hash", publicState }; + } return { ...base, outcome: "terminal_chain_rejection", terminal: true, reason: "bot tx reached terminal chain rejection", publicState }; } - if (phase === "pre_broadcast") { + if (phase === "pre_broadcast" && (failed["retryable"] !== true || failed["terminal"] !== false)) { return { ...base, outcome: "unknown", terminal: true, reason: "bot pre-broadcast transaction failure", publicState }; } } @@ -1163,6 +1347,17 @@ function classifyBotResult( }; } + const iterationFailure = lastRecordOfType(botRecords, "bot.iteration.failed"); + if (iterationFailure !== undefined && iterationFailure["retryable"] === true && iterationFailure["terminal"] === true) { + return { + ...base, + outcome: "bot_retryable_error", + terminal: true, + reason: "bot reported terminal retryable iteration failure", + publicState, + }; + } + if (result.status !== 0) { return { ...base, @@ -1173,12 +1368,15 @@ function classifyBotResult( }; } - const committed = lastRecordOfType(botRecords, "bot.transaction.committed"); - if (committed !== undefined) { + if (lifecycle?.type === "bot.transaction.committed") { + const committed = lifecycle.record; if (!hasValidTxHash(committed)) { return { ...base, outcome: "malformed_evidence", terminal: true, reason: "bot committed transaction evidence did not include a valid tx hash", publicState }; } - const actions = latestBotActions(botRecords) ?? emptyActions(); + const actions = latestBotActions(botRecords, numberField(committed, "iterationId")); + if (actions === undefined) { + return { ...base, outcome: "malformed_evidence", terminal: true, reason: "bot committed transaction evidence did not include matching built action evidence", publicState }; + } return { ...base, outcome: classifyBotCommittedActions(actions), @@ -1213,6 +1411,16 @@ function classifyBotResult( publicState, }; } + + if (iterationFailure !== undefined && iterationFailure["retryable"] === true && iterationFailure["terminal"] === false) { + return { + ...base, + outcome: "bot_retryable_error", + terminal: false, + reason: "bot reported retryable iteration failure", + publicState, + }; + } return { ...base, outcome: "unknown", terminal: true, reason: "bot produced no classifiable terminal evidence", publicState }; } @@ -1227,8 +1435,9 @@ function classifyTesterResult( if (latest !== undefined) { const error = latest["error"]; if (error !== undefined) { - if (hasTimeoutError(error)) { - return { ...base, outcome: "confirmation_timeout", terminal: true, reason: "tester transaction confirmation timed out" }; + const transactionFailure = classifyTesterTransactionFailure(error, latest); + if (transactionFailure !== undefined) { + return { ...base, ...transactionFailure }; } if (isTesterFundingError(error)) { return { ...base, outcome: "low_capital_stop", terminal: true, reason: "tester reported insufficient funds" }; @@ -1251,6 +1460,9 @@ function classifyTesterResult( if (reason === "fresh-matchable-order") { return { ...base, outcome: "tester_fresh_order_skip", terminal: false, reason: "tester skipped fresh matchable order", skipReason: reason }; } + if (reason === "matchable-order-transaction-missing") { + return { ...base, outcome: "tester_fresh_order_skip", terminal: false, reason: "tester skipped because matchable order transaction was not readable yet", skipReason: reason }; + } if (reason === "sampled-amount-too-small") { return { ...base, outcome: "tester_sampled_too_small_skip", terminal: false, reason: "tester sampled amount too small", skipReason: reason }; } @@ -1272,16 +1484,84 @@ function classifyTesterResult( if (expectationFailure !== undefined) { return { ...base, outcome: "tester_deterministic_pre_broadcast_error", terminal: true, reason: expectationFailure }; } + const testerOrder = testerOrderEvidence(actions); const conversionKind = stringField(recordField(actions ?? {}, "conversion"), "kind"); + if (testerOrder === undefined && conversionKind === undefined) { + return { ...base, outcome: "malformed_evidence", terminal: true, reason: "tester committed transaction evidence did not include action evidence" }; + } if (conversionKind !== undefined) { - return { ...base, outcome: "tester_conversion_created", terminal: false, reason: "tester created a direct conversion transaction" }; + return { + ...base, + outcome: "tester_conversion_created", + terminal: false, + reason: "tester created a direct conversion transaction", + ...(testerOrder === undefined ? {} : { testerOrder }), + }; } - return { ...base, outcome: "tester_order_created", terminal: false, reason: "tester created an order transaction" }; + return { + ...base, + outcome: "tester_order_created", + terminal: false, + reason: "tester created an order transaction", + ...(testerOrder === undefined ? {} : { testerOrder }), + }; } } return { ...base, outcome: "unknown", terminal: true, reason: "tester produced no classifiable terminal evidence" }; } +function testerOrderEvidence(actions: Record | undefined): TesterOrderEvidence | undefined { + if (actions === undefined) { + return undefined; + } + const orders = testerOrderSummaries(actions); + if (orders.length === 0) { + return undefined; + } + return { + ...(stringField(actions, "requestedTesterScenario") === undefined ? {} : { requestedTesterScenario: stringField(actions, "requestedTesterScenario") }), + ...(stringField(actions, "testerScenario") === undefined ? {} : { testerScenario: stringField(actions, "testerScenario") }), + orderCount: numberField(actions, "orderCount") ?? orders.length, + ...(numberField(actions, "collectedOrders") === undefined ? {} : { collectedOrders: numberField(actions, "collectedOrders") }), + ...(numberField(actions, "cancelledOrders") === undefined ? {} : { cancelledOrders: numberField(actions, "cancelledOrders") }), + orders, + }; +} + +function testerOrderSummaries(actions: Record): TesterOrderSummary[] { + const newOrders = arrayField(actions, "newOrders"); + if (newOrders !== undefined) { + return newOrders.flatMap((order) => { + const summary = testerOrderSummary(order); + return summary === undefined ? [] : [summary]; + }); + } + const newOrder = recordField(actions, "newOrder"); + const summary = testerOrderSummary(newOrder); + return summary === undefined ? [] : [summary]; +} + +function testerOrderSummary(order: unknown): TesterOrderSummary | undefined { + if (!isRecord(order)) { + return undefined; + } + const giveCkb = stringField(order, "giveCkb"); + const takeIckb = stringField(order, "takeIckb"); + const giveIckb = stringField(order, "giveIckb"); + const takeCkb = stringField(order, "takeCkb"); + return { + ...(orderDirection(order) === undefined ? {} : { direction: orderDirection(order) }), + ...(giveCkb === undefined ? {} : { giveCkb }), + ...(takeIckb === undefined ? {} : { takeIckb }), + ...(giveIckb === undefined ? {} : { giveIckb }), + ...(takeCkb === undefined ? {} : { takeCkb }), + ...(stringField(order, "fee") === undefined ? {} : { fee: stringField(order, "fee") }), + ...(stringField(order, "feeNumerator") === undefined ? {} : { feeNumerator: stringField(order, "feeNumerator") }), + ...(stringField(order, "feeBase") === undefined ? {} : { feeBase: stringField(order, "feeBase") }), + dust: giveCkb === "0.00000001" || giveIckb === "0.00000001", + }; +} + function validateTesterEvidenceExpectation(actions: Record | undefined, expectation: TesterEvidenceExpectation | undefined): string | undefined { if (expectation === undefined) { return undefined; @@ -1398,35 +1678,35 @@ function isSdkConversionTesterScenario(scenario: TesterScenario): boolean { } function isIckbToCkbTesterScenario(scenario: TesterScenario): boolean { - return scenario === "ickb-to-ckb-limit-order" || scenario === "two-ickb-to-ckb-limit-orders" || scenario === "dust-ickb-conversion"; + return scenario === "ickb-to-ckb-limit-order" || scenario === "bounded-ickb-to-ckb-limit-order" || scenario === "two-ickb-to-ckb-limit-orders" || scenario === "dust-ickb-conversion"; } function classifyBotCommittedActions(actions: ActionCounts): OutcomeKind { if (actions.matchedOrders > 0 && actions.deposits > 0) { return "bot_match_plus_deposit_committed"; } - if (actions.matchedOrders > 0) { - return "bot_match_committed"; + if (actions.withdrawalRequests > 0) { + return "bot_withdrawal_request_committed"; } if (actions.completedDeposits > 0) { return "bot_receipt_completion_committed"; } - if (actions.deposits > 0) { - return "bot_deposit_only_committed"; - } - if (actions.withdrawalRequests > 0) { - return "bot_withdrawal_request_committed"; - } if (actions.withdrawals > 0) { return "bot_withdrawal_completion_committed"; } + if (actions.matchedOrders > 0) { + return "bot_match_committed"; + } + if (actions.deposits > 0) { + return "bot_deposit_only_committed"; + } return "unknown"; } function unsupportedClassification(choice: UnsupportedScenarioChoice): Classification { return { actor: "preflight", - outcome: "unknown", + outcome: "unsupported_scenario", terminal: true, reason: choice.reason, txHashes: [], @@ -1437,6 +1717,8 @@ function unsupportedClassification(choice: UnsupportedScenarioChoice): Classific exitStatus: null, signal: null, timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, }, }; } @@ -1456,12 +1738,13 @@ async function writeUnmetCoverageIncident( ledger: CoverageLedger, artifacts: string[], dependencies: Dependencies, + budgetLabel: string, ): Promise { const classification: Classification = { actor: "preflight", outcome: "unmet_coverage_goal", terminal: true, - reason: `bounded cycle budget ended before observing requested outcomes: ${unmetGoals.join(", ")}`, + reason: `${budgetLabel} ended before observing requested outcomes: ${unmetGoals.join(", ")}`, txHashes: [], evidence: { recordsAccepted: 0, @@ -1470,6 +1753,8 @@ async function writeUnmetCoverageIncident( exitStatus: null, signal: null, timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, }, }; const relativePath = await writeJsonArtifact(plan, `cycle-${padCycle(cycleIndex)}-incident.json`, { @@ -1502,7 +1787,7 @@ async function writeIncident( actor, scenario: choice.scenario.name, targetOutcomes: choice.targetOutcomes, - command: redactedCommandShape(plan, result), + command: commandShape(plan, result), exit: { spawnError: result.spawnError, status: result.status, @@ -1514,8 +1799,8 @@ async function writeIncident( }, classification, coverage: coverageSummary(ledger), - stdoutExcerpt: boundedText(safeArtifactText(result.stdout), 4000), - stderrExcerpt: boundedText(safeArtifactText(result.stderr), 4000), + stdoutExcerpt: boundedText(result.stdout, 4000), + stderrExcerpt: boundedText(result.stderr, 4000), artifacts, suggestedNextAction: suggestedNextAction(classification), }, artifacts, dependencies); @@ -1545,6 +1830,7 @@ async function writeSummary( ledger: CoverageLedger, classifications: Classification[], artifacts: string[], + preflightState: PreflightStateSummary[], latestPublicState: PublicStateAssumption | undefined, stopReason: string, dependencies: Dependencies, @@ -1555,7 +1841,11 @@ async function writeSummary( artifacts, aggregateCounts: aggregateClassifications(classifications), txHashesByOutcome: txHashesByOutcome(classifications), + txCreatingTxHashCount: txCreatingHashCountTotal(classifications), + txCreatingOutcomeCount: txCreatingOutcomeCount(classifications), + testerOrderEvidence: testerOrderEvidenceByOutcome(classifications), skipReasons: classifications.map((item) => item.skipReason).filter((item) => item !== undefined), + preflightState, scenarioAttempts: ledger.attempts, coverage: coverageSummary(ledger), publicVsOwnedStateAssumptions: latestPublicState ?? null, @@ -1570,10 +1860,11 @@ async function writeCommandArtifacts( dependencies: Dependencies, ): Promise { const base = `cycle-${padCycle(cycleIndex)}-${label}`; - const stdoutPath = await writeTextArtifact(plan, `${base}.stdout.ndjson`, safeArtifactText(result.stdout), dependencies); - const stderrPath = await writeTextArtifact(plan, `${base}.stderr.log`, safeArtifactText(result.stderr), dependencies); + const stdoutExtension = result.actor === "preflight" ? "json" : "ndjson"; + const stdoutPath = await writeTextArtifact(plan, `${base}.stdout.${stdoutExtension}`, result.stdout, dependencies); + const stderrPath = await writeTextArtifact(plan, `${base}.stderr.log`, result.stderr, dependencies); const commandPath = await writeJsonArtifact(plan, `${base}.command.json`, { - command: redactedCommandShape(plan, result), + command: commandShape(plan, result), exit: { spawnError: result.spawnError, status: result.status, @@ -1591,7 +1882,7 @@ async function writeTextArtifact(plan: SupervisorPlan, fileName: string, text: s const writeFileFn = dependencies.writeFile ?? writeFile; const artifactPath = join(plan.outDir, fileName); await writeFileFn(artifactPath, text); - return relative(plan.rootDir, artifactPath); + return displayPath(plan.rootDir, artifactPath); } async function writeJsonArtifact( @@ -1604,7 +1895,7 @@ async function writeJsonArtifact( const writeFileFn = dependencies.writeFile ?? writeFile; const artifactPath = join(plan.outDir, fileName); await writeFileFn(artifactPath, `${JSON.stringify(value, jsonReplacer, 2)}\n`); - const relativePath = relative(plan.rootDir, artifactPath); + const relativePath = displayPath(plan.rootDir, artifactPath); if (!artifacts.includes(relativePath)) { artifacts.push(relativePath); } @@ -1622,7 +1913,7 @@ async function appendSupervisorEvent(plan: SupervisorPlan, fields: Record { +function commandShape(plan: SupervisorPlan, result: CommandResult): Record { return { command: result.command === process.execPath ? "node" : result.command, args: result.args.map((arg) => arg === plan.botConfigPath @@ -1655,10 +1946,28 @@ function txHashesByOutcome(classifications: Classification[]): Record { + return classifications.flatMap((classification) => classification.testerOrder === undefined + ? [] + : [{ + outcome: classification.outcome, + txHashes: classification.txHashes, + ...classification.testerOrder, + }]); +} + function txCreatingHashCount(classification: Classification): number { return TX_CREATING_OUTCOMES.has(classification.outcome) ? classification.txHashes.length : 0; } +function txCreatingHashCountTotal(classifications: Classification[]): number { + return classifications.reduce((sum, classification) => sum + txCreatingHashCount(classification), 0); +} + +function txCreatingOutcomeCount(classifications: Classification[]): number { + return classifications.filter((classification) => TX_CREATING_OUTCOMES.has(classification.outcome)).length; +} + function coverageSummary(ledger: CoverageLedger): Record { return { goals: ledger.goals, @@ -1674,22 +1983,19 @@ function suggestedNextAction(classification: Classification): string { if (classification.outcome === "confirmation_timeout" || classification.outcome === "post_broadcast_unresolved") { return "confirm the tx hash with a read-only chain query before sending any follow-up work"; } - if (classification.outcome === "secret_leak_sentinel") { - return "stop, inspect artifacts for leakage, and rotate any exposed disposable key before relaunch"; - } if (classification.outcome === "low_capital_stop") { return "fund the supervised account or provide an alternate ignored config, then rerun a bounded smoke"; } return "inspect the incident bundle and run a review pass for material code changes before extended relaunch"; } -function latestBotActions(records: Record[]): ActionCounts | undefined { +function latestBotActions(records: Record[], iterationId?: number): ActionCounts | undefined { for (let index = records.length - 1; index >= 0; index -= 1) { const record = records[index]; if (record === undefined) { continue; } - if (stringField(record, "type") === "bot.transaction.built") { + if (stringField(record, "type") === "bot.transaction.built" && (iterationId === undefined || numberField(record, "iterationId") === iterationId)) { return actionCounts(record["actions"]); } } @@ -1763,41 +2069,27 @@ function emptyActions(): ActionCounts { function extractTxHashes(records: Record[]): string[] { const hashes = new Set(); for (const record of records) { - const txHash = record["txHash"]; - if (typeof txHash === "string" && TX_HASH_PATTERN.test(txHash)) { - hashes.add(txHash); - } + addValidTxHash(hashes, record["txHash"]); + addValidTxHash(hashes, recordField(record, "error")?.["txHash"]); const skip = recordField(record, "skip"); - const skipTxHash = skip?.["txHash"]; - if (typeof skipTxHash === "string" && TX_HASH_PATTERN.test(skipTxHash)) { - hashes.add(skipTxHash); - } + addValidTxHash(hashes, skip?.["txHash"]); } return [...hashes]; } +function addValidTxHash(hashes: Set, value: unknown): void { + if (typeof value === "string" && TX_HASH_PATTERN.test(value)) { + hashes.add(value); + } +} + function hasValidTxHash(record: Record): boolean { const txHash = record["txHash"]; return typeof txHash === "string" && TX_HASH_PATTERN.test(txHash); } -function containsSecretLeak(text: string): boolean { - return /["']?(private[-_]?key|mnemonic|seed[-_]?phrase)["']?\s*[:=]/iu.test(text) || - /["']?rpc[-_]?url["']?\s*[:=]\s*["']?https?:\/\/[^\s"']*(?:@|[?&][^\s"']*=)/iu.test(text); -} - -function containsTransactionLeak(text: string): boolean { - return /["']?(witnesses|cellDeps|headerDeps|inputs|outputs|outputsData)["']?\s*:\s*\[/iu.test(text); -} - -export function safeArtifactText(text: string): string { - if (containsSecretLeak(text)) { - return "\n"; - } - if (containsTransactionLeak(text)) { - return "\n"; - } - return text; +function hasValidTxHashInRecordOrError(record: Record): boolean { + return hasValidTxHash(record) || hasValidTxHash(recordField(record, "error") ?? {}); } function minimalProcessEnv(env: NodeJS.ProcessEnv): Record { @@ -1810,15 +2102,28 @@ function minimalProcessEnv(env: NodeJS.ProcessEnv): Record { function liveActorEnv(extra: Record): Record { return { ...minimalProcessEnv(process.env), + NODE_OPTIONS: "--disable-warning=DEP0040", ...extra, }; } -function hasTimeoutError(value: unknown): boolean { +function classifyTesterTransactionFailure(value: unknown, record: Record): Pick | undefined { if (!isRecord(value)) { - return false; + return undefined; + } + if (stringField(value, "name") !== "TransactionConfirmationError") { + return undefined; + } + if (!hasValidTxHashInRecordOrError(record)) { + return { outcome: "malformed_evidence", terminal: true, reason: "tester transaction failure evidence did not include a valid tx hash" }; } - return value["isTimeout"] === true || stringField(value, "name") === "TransactionConfirmationError"; + if (value["isTimeout"] === false) { + return { outcome: "terminal_chain_rejection", terminal: true, reason: "tester tx reached terminal chain rejection" }; + } + if (value["isTimeout"] === true && "cause" in value) { + return { outcome: "post_broadcast_unresolved", terminal: true, reason: "tester tx remained unresolved after broadcast" }; + } + return { outcome: "confirmation_timeout", terminal: true, reason: "tester transaction confirmation timed out" }; } function isTesterFundingError(value: unknown): boolean { @@ -1843,6 +2148,20 @@ function lastRecordOfType(records: Record[], type: string): Rec return undefined; } +function lastRecordOfTypes(records: Record[], types: readonly string[]): { type: string; record: Record } | undefined { + for (let index = records.length - 1; index >= 0; index -= 1) { + const record = records[index]; + if (record === undefined) { + continue; + } + const type = stringField(record, "type"); + if (type !== undefined && types.includes(type)) { + return { type, record }; + } + } + return undefined; +} + function recordField(record: Record, key: string): Record | undefined { const value = record[key]; return isRecord(value) ? value : undefined; @@ -1898,6 +2217,7 @@ function parseTesterScenario(value: string): TesterScenario { value === "two-ckb-to-ickb-limit-orders" || value === "all-ckb-limit-order" || value === "ickb-to-ckb-limit-order" || + value === "bounded-ickb-to-ckb-limit-order" || value === "two-ickb-to-ckb-limit-orders" || value === "mixed-direction-limit-orders" || value === "dust-ckb-conversion" || @@ -1905,7 +2225,7 @@ function parseTesterScenario(value: string): TesterScenario { ) { return value; } - throw new Error("Invalid --tester-scenario: expected auto, random-order, sdk-conversion, extra-large-limit-order, multi-order-limit-orders, two-ckb-to-ickb-limit-orders, all-ckb-limit-order, ickb-to-ckb-limit-order, two-ickb-to-ckb-limit-orders, mixed-direction-limit-orders, dust-ckb-conversion, or dust-ickb-conversion"); + throw new Error("Invalid --tester-scenario: expected auto, random-order, sdk-conversion, extra-large-limit-order, multi-order-limit-orders, two-ckb-to-ickb-limit-orders, all-ckb-limit-order, ickb-to-ckb-limit-order, bounded-ickb-to-ckb-limit-order, two-ickb-to-ckb-limit-orders, mixed-direction-limit-orders, dust-ckb-conversion, or dust-ickb-conversion"); } function valueAfter(argv: string[], index: number, option: string): string { @@ -1934,11 +2254,15 @@ function parseTesterFeeValue(value: string, flag: string): string { return value; } -function insideRepoPath(rootDir: string, path: string, label: string): string { +function resolveConfiguredPath(rootDir: string, path: string, label: string): string { if (path === "") { throw new Error(`${label} must not be empty`); } - const absolutePath = isAbsolute(path) ? path : resolve(rootDir, path); + return isAbsolute(path) ? resolve(path) : resolve(rootDir, path); +} + +function insideRepoPath(rootDir: string, path: string, label: string): string { + const absolutePath = resolveConfiguredPath(rootDir, path, label); const relativePath = relative(rootDir, absolutePath); if (relativePath.startsWith("..") || isAbsolute(relativePath)) { throw new Error(`${label} must stay inside the repo`); @@ -1946,11 +2270,31 @@ function insideRepoPath(rootDir: string, path: string, label: string): string { return absolutePath; } -function assertSupervisorOutputDirectory(relativeOutDir: string): void { - if (relativeOutDir === "logs/live-supervisor" || relativeOutDir.startsWith(SUPERVISOR_OUTPUT_ROOT)) { +function assertSupervisorOutputDirectory(outDir: string, relativeOutDir: string): void { + if ( + relativeOutDir === "logs/live-supervisor" || + relativeOutDir.startsWith(SUPERVISOR_OUTPUT_ROOT) || + isValidationRunOutputDirectory(outDir) + ) { return; } - throw new Error(`Supervisor output directory must be under ${SUPERVISOR_OUTPUT_ROOT}`); + throw new Error(`Supervisor output directory must be under ${SUPERVISOR_OUTPUT_ROOT} or a validation session run directory`); +} + +function isValidationRunOutputDirectory(path: string): boolean { + const parts = path.split(/[\\/]+/u); + for (let index = 0; index < parts.length - 4; index += 1) { + if ( + parts[index] === "validation" && + parts[index + 2] === "chunks" && + /^chunk-[0-9]{4}$/u.test(parts[index + 3] ?? "") && + /^run-[0-9]{4}$/u.test(parts[index + 4] ?? "") && + index + 5 === parts.length + ) { + return true; + } + } + return false; } function assertIgnoredConfigPath( @@ -1998,6 +2342,15 @@ function isIgnoredPath(rootDir: string, relativePath: string, dependencies: Depe return result.status === 0; } +function isInside(rootDir: string, path: string): boolean { + const relativePath = relative(rootDir, path); + return relativePath === "" || (!relativePath.startsWith("..") && !isAbsolute(relativePath)); +} + +function displayPath(rootDir: string, path: string): string { + return isInside(rootDir, path) ? relative(rootDir, path) : path; +} + export function createBoundedOutputCapture(): BoundedOutputCapture { return { chunks: [], byteLength: 0, truncatedBytes: 0 }; } From 2db7d688c9e5a742236e1513b68411f0a9d84524 Mon Sep 17 00:00:00 2001 From: phroi <90913182+phroi@users.noreply.github.com> Date: Thu, 28 May 2026 19:44:18 +0000 Subject: [PATCH 2/7] feat(supervisor): add dynamic loop tooling --- .gitignore | 1 + package.json | 1 + scripts/ickb-script-helpers.mjs | 31 ++ scripts/ickb-supervisor-dynamic-loop.mjs | 524 ++++++++++++++++++ scripts/ickb-supervisor-dynamic-loop.test.mjs | 424 ++++++++++++++ scripts/ickb-supervisor-loop.mjs | 178 +++--- scripts/ickb-supervisor-loop.test.mjs | 274 +++++++-- 7 files changed, 1316 insertions(+), 117 deletions(-) create mode 100644 scripts/ickb-script-helpers.mjs create mode 100644 scripts/ickb-supervisor-dynamic-loop.mjs create mode 100644 scripts/ickb-supervisor-dynamic-loop.test.mjs diff --git a/.gitignore b/.gitignore index 72043ac..57af644 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ # Artifacts **/dist/ +log/ apps/*/log_*.json # Local runtime config files diff --git a/package.json b/package.json index a1a93ed..c7f8ca3 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "live:preflight": "node scripts/ickb-live-preflight.mjs", "live:supervisor": "node apps/supervisor/dist/index.js", "live:supervisor:loop": "node scripts/ickb-supervisor-loop.mjs", + "live:supervisor:dynamic-loop": "node scripts/ickb-supervisor-dynamic-loop.mjs", "coworker:ask": "opencode run --pure --agent plan" }, "engines": { diff --git a/scripts/ickb-script-helpers.mjs b/scripts/ickb-script-helpers.mjs new file mode 100644 index 0000000..4001e3d --- /dev/null +++ b/scripts/ickb-script-helpers.mjs @@ -0,0 +1,31 @@ +const MAX_SAFE_INTEGER = BigInt(Number.MAX_SAFE_INTEGER); + +export function valueAfter(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith("--")) { + throw new Error(`Missing value for ${flag}`); + } + return value; +} + +export function parsePositiveInteger(value, flag) { + if (!/^[1-9][0-9]*$/u.test(value)) { + throw new Error(`Invalid ${flag}: expected a positive integer`); + } + const parsed = BigInt(value); + if (parsed > MAX_SAFE_INTEGER) { + throw new Error(`Invalid ${flag}: expected a safe integer`); + } + return Number(parsed); +} + +export function parseNonNegativeInteger(value, flag) { + if (!/^(0|[1-9][0-9]*)$/u.test(value)) { + throw new Error(`Invalid ${flag}: expected a non-negative integer`); + } + const parsed = BigInt(value); + if (parsed > MAX_SAFE_INTEGER) { + throw new Error(`Invalid ${flag}: expected a safe integer`); + } + return Number(parsed); +} diff --git a/scripts/ickb-supervisor-dynamic-loop.mjs b/scripts/ickb-supervisor-dynamic-loop.mjs new file mode 100644 index 0000000..a968aca --- /dev/null +++ b/scripts/ickb-supervisor-dynamic-loop.mjs @@ -0,0 +1,524 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { appendFile, lstat, mkdir, stat, writeFile } from "node:fs/promises"; +import { isAbsolute, join, parse, relative, resolve, sep } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { parseNonNegativeInteger, parsePositiveInteger, valueAfter } from "./ickb-script-helpers.mjs"; +import { DEFAULT_CHILD_TIMEOUT_SECONDS_VALUE as DEFAULT_SUPERVISOR_LOOP_CHILD_TIMEOUT_SECONDS } from "./ickb-supervisor-loop.mjs"; + +const rootDir = fileURLToPath(new URL("..", import.meta.url)); +const CKB_UNIT = 100000000n; +const DEFAULT_TESTER_CONFIG = "config/tester-testnet.json"; +const DEFAULT_PREFLIGHT_SCRIPT = "scripts/ickb-live-preflight.mjs"; +const DEFAULT_SUPERVISOR_LOOP_SCRIPT = "scripts/ickb-supervisor-loop.mjs"; +const DEFAULT_LOG_ROOT = "log"; +const DEFAULT_CHUNK_MAX_RUNS = 8; +const DEFAULT_STABLE_LIMIT = 999; +const DEFAULT_CHUNK_BACKOFF_SECONDS = 20; +const DEFAULT_BETWEEN_CHUNKS_SECONDS = 20; +const DEFAULT_CHILD_TIMEOUT_SECONDS = DEFAULT_SUPERVISOR_LOOP_CHILD_TIMEOUT_SECONDS; +export const DEFAULT_CHILD_TIMEOUT_SECONDS_VALUE = DEFAULT_CHILD_TIMEOUT_SECONDS; +const DEFAULT_COMMAND_TIMEOUT_SECONDS = 240; +const DEFAULT_CHUNK_TIMEOUT_MARGIN_SECONDS = 60; +const DEFAULT_PREFLIGHT_TIMEOUT_SECONDS = 120; +const ALL_CKB_MIN_CKB = 3001n; +const ICKB_STIMULUS_MIN_CKB = 2100n; + +export function parseArgs(argv) { + const args = { + help: false, + testerConfig: DEFAULT_TESTER_CONFIG, + preflightRole: "tester-dynamic-loop", + preflightScript: DEFAULT_PREFLIGHT_SCRIPT, + supervisorLoopScript: DEFAULT_SUPERVISOR_LOOP_SCRIPT, + chunkMaxRuns: DEFAULT_CHUNK_MAX_RUNS, + stableLimit: DEFAULT_STABLE_LIMIT, + chunkBackoffSeconds: DEFAULT_CHUNK_BACKOFF_SECONDS, + betweenChunksSeconds: DEFAULT_BETWEEN_CHUNKS_SECONDS, + childTimeoutSeconds: DEFAULT_CHILD_TIMEOUT_SECONDS, + commandTimeoutSeconds: DEFAULT_COMMAND_TIMEOUT_SECONDS, + preflightTimeoutSeconds: DEFAULT_PREFLIGHT_TIMEOUT_SECONDS, + supervisorArgs: [], + }; + let chunkTimeoutSecondsExplicit = false; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--") { + args.supervisorArgs = argv.slice(index + 1); + break; + } + if (arg === "-h" || arg === "--help") { + args.help = true; + continue; + } + if (arg === "--tester-config") { + args.testerConfig = valueAfter(argv, ++index, arg); + continue; + } + if (arg === "--preflight-role") { + args.preflightRole = valueAfter(argv, ++index, arg); + continue; + } + if (arg === "--preflight-script") { + args.preflightScript = valueAfter(argv, ++index, arg); + continue; + } + if (arg === "--supervisor-loop-script") { + args.supervisorLoopScript = valueAfter(argv, ++index, arg); + continue; + } + if (arg === "--log-root") { + args.logRoot = valueAfter(argv, ++index, arg); + continue; + } + if (arg === "--session-root") { + args.sessionRoot = valueAfter(argv, ++index, arg); + continue; + } + if (arg === "--max-chunks") { + args.maxChunks = parsePositiveInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + if (arg === "--chunk-max-runs") { + args.chunkMaxRuns = parsePositiveInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + if (arg === "--stable-limit") { + args.stableLimit = parsePositiveInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + if (arg === "--chunk-backoff-seconds") { + args.chunkBackoffSeconds = parseNonNegativeInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + if (arg === "--between-chunks-seconds") { + args.betweenChunksSeconds = parseNonNegativeInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + if (arg === "--child-timeout-seconds") { + args.childTimeoutSeconds = parsePositiveInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + if (arg === "--command-timeout-seconds") { + args.commandTimeoutSeconds = parsePositiveInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + if (arg === "--chunk-timeout-seconds") { + args.chunkTimeoutSeconds = parsePositiveInteger(valueAfter(argv, ++index, arg), arg); + chunkTimeoutSecondsExplicit = true; + continue; + } + if (arg === "--preflight-timeout-seconds") { + args.preflightTimeoutSeconds = parsePositiveInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + if (args.supervisorArgs.some((arg) => arg === "--out-dir" || arg.startsWith("--out-dir="))) { + throw new Error("Do not pass supervisor --out-dir; dynamic-loop owns the session chunk roots"); + } + const minimumChunkTimeoutSeconds = supervisorLoopChunkTimeoutFloorSeconds(args); + if (chunkTimeoutSecondsExplicit) { + if (BigInt(args.chunkTimeoutSeconds) < minimumChunkTimeoutSeconds) { + throw new Error(`Invalid --chunk-timeout-seconds: expected at least ${minimumChunkTimeoutSeconds.toString()} seconds for this chunk shape`); + } + } else { + args.chunkTimeoutSeconds = Number(minimumChunkTimeoutSeconds); + } + return args; +} + +export function usage() { + return [ + "Usage: node scripts/ickb-supervisor-dynamic-loop.mjs [options]", + "Options:", + ` --tester-config Default: ${DEFAULT_TESTER_CONFIG}`, + ` --preflight-role